From 0cebde9bd7337e7cf26b9b6f0dbdf0b5139310ad Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Fri, 7 Aug 2026 13:00:41 -0500 Subject: [PATCH 01/48] A more generic back end expand via a new /v1/id/_:id/expanded/ endpoint --- controllers/crud.js | 167 ++++++++++++++++- controllers/gog.js | 69 +------ controllers/utils.js | 64 +++++++ database/__mocks__/index.js | 1 + db-controller.js | 3 +- openapi/contracts/core-provider.openapi.yaml | 93 ++++++++++ public/API.html | 186 +++++++++++++++++++ routes/id.js | 9 + 8 files changed, 528 insertions(+), 64 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index 758a048a..bcadc537 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -6,7 +6,7 @@ */ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js' +import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID, findLeafAnnotationsFor } from './utils.js' /** * Create a new Linked Open Data object in RERUM v1. @@ -127,8 +127,171 @@ const id = async function (req, res, next) { } } +/** + * The expand job always constrains the Annotations it gathers to the leaf versions, to the + * Annotation types, and to the entity in the request URI. A client cannot influence those, so + * these keys are dropped from a supplied filter body by exact name or dotted prefix. + * The dot matters -- 'targetCollection' is a real property on Gallery of Glosses data and must + * still be usable as a filter. + */ +const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"] + +/** + * Identity and system properties an Annotation body must never overwrite. '@id' and '@context' + * are read by idNegotiation() and res.location() right after the merge, so clobbering them would + * break the response itself. '__proto__' is not data -- assigning it would re-point the response + * object's prototype instead of adding a property, and emitting it would hand a prototype + * pollution vector to every client that parses the response. + */ +const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"]) + +/** + * Reduce a supplied POST body to the literal MongoDB filter keys the expand job will honor. + * @param supplied The parsed JSON request body. + * @return An object of filter keys, minus the ones this endpoint owns. + */ +function sanitizeExpansionFilters(supplied) { + const filters = {} + for (const [key, value] of Object.entries(supplied)) { + if (RESERVED_FILTER_KEYS.some(reserved => key === reserved || key.startsWith(`${reserved}.`))) continue + filters[key] = value + } + return filters +} + +/** + * The [key, value] assertions an Annotation makes about the entity it targets. + * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other + * property of the Annotation can leak onto the entity. + * + * Anticipates the likely Annotation body formats + * - bodyValue: 'text' the W3C shorthand, which has no key of its own + * - body: {'key': 'value'} a single assertion + * - body: {'key': {...}} a single assertion, value kept as-is + * - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive + * + * @param anno An Annotation document. + * @return An Array of [key, value] pairs to merge onto the entity. + */ +function assertionsFrom(anno) { + const assertions = [] + if (typeof anno.bodyValue === "string") assertions.push(["bodyValue", anno.bodyValue]) + const body = anno.body + // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an + // external resource with no embedded value to expand with. + if (!body || typeof body !== "object" || Array.isArray(body)) return assertions + if ((body.type ?? body["@type"]) === "TextualBody") { + assertions.push(["bodyValue", body]) + return assertions + } + const keys = Object.keys(body) + // Any other multi-key body is structural rather than assertional and cannot be attributed to a + // single entity property. This is what skips the Choice, Composite, and List multiplicity + // constructs, which are all shaped {type, items}. + if (keys.length !== 1) return assertions + assertions.push([keys[0], body[keys[0]]]) + return assertions +} + +/** + * Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values. + * Unlike the Gallery of Glosses expand(), values are not wrapped and not unwrapped -- what the + * Annotation says is what the entity gets. When more than one current Annotation asserts the same + * key, or the entity already carries it, the values collect into an Array. + * @param primitiveEntity The unexpanded entity. + * @param annos The Annotations targeting it. + * @return A new, expanded entity object. + */ +function applyRawExpansion(primitiveEntity, annos) { + const expandedEntity = structuredClone(primitiveEntity) + // Hold __rerum aside so it can be re-appended after the merged properties. It is the + // last property on a stored object and should stay last on an expanded one. + const rerumProp = expandedEntity.__rerum + delete expandedEntity.__rerum + for (const anno of annos) { + for (const [key, value] of assertionsFrom(anno)) { + if (PROTECTED_EXPANSION_KEYS.has(key)) continue + if (!Object.hasOwn(expandedEntity, key)) { + expandedEntity[key] = value + continue + } + const existing = Array.isArray(expandedEntity[key]) ? expandedEntity[key] : [expandedEntity[key]] + expandedEntity[key] = Array.isArray(value) ? [...existing, ...value] : [...existing, value] + } + } + if (rerumProp !== undefined) expandedEntity.__rerum = rerumProp + return expandedEntity +} + +/** + * Query the MongoDB for the object with the _id or __rerum.slug provided in the request URL, then + * merge in the assertions of all the current Annotations targeting it. + * + * GET recognizes the '?generator=' and '?creator=' convenience parameters only. + * POST reads literal MongoDB filter keys from the JSON body and ignores URL parameters as filters. + * Both methods page the Annotation search with the usual '?limit=' and '?skip=' parameters. + * */ +const idExpanded = async function (req, res, next) { + res.set("Content-Type", "application/json; charset=utf-8") + const id = req.params["_id"] + const isPost = req.method === "POST" + //Paging is transport rather than a filter, so it comes off the URL for both methods. + //The default is generous because an expansion wants every Annotation it can get, and an + //entity with more than 200 targeting it is not expected. + const pagination = getPagination(req.query, 200) + let filters = {} + if (isPost) { + //Express leaves the body undefined when a POST supplies none. That is an unfiltered expand. + const supplied = req.body ?? {} + if (typeof supplied !== "object" || Array.isArray(supplied)) { + const err = { + "message": "The /expanded request body must be a JSON object of filter properties.", + "status": 400 + } + return next(utils.createExpressError(err)) + } + filters = sanitizeExpansionFilters(supplied) + } + else { + //Repeated query parameters arrive as an Array, which is not a filter value we support. + if (typeof req.query.generator === "string" && req.query.generator) filters["__rerum.generatedBy"] = req.query.generator + if (typeof req.query.creator === "string" && req.query.creator) filters.creator = req.query.creator + } + try { + const match = await db.findOne({"$or": [{"_id": id}, {"__rerum.slug": id}]}) + if (!match) { + const err = { + "message": `No RERUM object with id '${id}'`, + "status": 404 + } + return next(utils.createExpressError(err)) + } + res.set(utils.configureWebAnnoHeadersFor(match)) + //Support built in browser caching. A POST response is not cacheable. + if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate") + // No Last-Modified here, unlike GET /v1/id/:_id. It would compare against the root entity + // before the targeting Annotations are merged in. + // Include current version for optimistic locking + res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "") + // Annotations target the stored URI, so this must come off the raw match. idNegotiation() + // below rebuilds 'id' from RERUM_ID_PREFIX, which is not necessarily the stored host. + const targetId = match["@id"] ?? match.id + const annos = targetId ? await findLeafAnnotationsFor(targetId, filters, pagination) : [] + // Let clients detect a full page. When this equals the limit there may be more to gather, + // and the entity in hand is expanded from only part of its Annotations. + res.set('Annotations-Merged', String(annos.length)) + let expanded = applyRawExpansion(match, annos) + expanded = idNegotiation(expanded) + res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) + res.json(expanded) + } catch (error) { + return next(utils.createExpressError(error)) + } +} + export { create, query, - id + id, + idExpanded } diff --git a/controllers/gog.js b/controllers/gog.js index db69a3e5..bfbe1b7c 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -8,7 +8,7 @@ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation } from './utils.js' +import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor } from './utils.js' // The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different // agents; only the trailing id is compared, so either host spelling matches. @@ -336,66 +336,13 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde // An entity is expandable if it carries a URI under either '@id' or 'id'. if(!primitiveEntity?.["@id"] && !primitiveEntity?.id) return primitiveEntity const targetId = primitiveEntity["@id"] ?? primitiveEntity.id ?? "unknown" - // '$and' is always present so the GENERATOR and CREATOR blocks below can push into it from - // either branch. 'annoTypeConditions' is always pushed, so it is never the empty Array Mongo rejects. - let queryObj = { - "__rerum.history.next": { $exists: true, $size: 0 }, - "$and": [] - } - let targetPatterns = ["target", "target.@id", "target.id"] - let targetConditions = [] - let annoTypeConditions = [{"type": "Annotation"}, {"@type":"Annotation"}, {"@type":"oa:Annotation"}] - - if (targetId.startsWith("http")) { - for(const targetKey of targetPatterns){ - targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) - targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) - } - queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) - } - else{ - queryObj["$and"].push({"$or": annoTypeConditions}) - queryObj.target = targetId - } - - // Only expand with data from a specific app - if(GENERATOR) { - // Need to check http:// and https:// - const generatorConditions = [ - {"__rerum.generatedBy": GENERATOR.replace(/^https?/, "http")}, - {"__rerum.generatedBy": GENERATOR.replace(/^https?/, "https")} - ] - if (GENERATOR.startsWith("http")) { - queryObj["$and"].push({"$or": generatorConditions }) - } - else{ - // It should be a URI, but this can be a fallback. - queryObj["__rerum.generatedBy"] = GENERATOR - } - } - // Only expand with data from a specific creator - if(CREATOR) { - // Need to check http:// and https:// - const creatorConditions = [ - {"creator": CREATOR.replace(/^https?/, "http")}, - {"creator": CREATOR.replace(/^https?/, "https")} - ] - if (CREATOR.startsWith("http")) { - queryObj["$and"].push({"$or": creatorConditions }) - } - else{ - // It should be a URI, but this can be a fallback. - queryObj["creator"] = CREATOR - } - } - - // Get the Annotations targeting this Entity from the db. Remove _id property. - // Assuming we do not need paged query here - let matches = await db.find(queryObj).toArray() - matches = matches.map(o => { - delete o._id - return o - }) + // Only expand with data from a specific app and/or a specific creator. The shared helper + // applies the leaf, target, and Annotation type constraints and doubles these two URIs + // across the http/https spellings. + const filters = {} + if(GENERATOR) filters["__rerum.generatedBy"] = GENERATOR + if(CREATOR) filters.creator = CREATOR + const matches = await findLeafAnnotationsFor(targetId, filters) // Combine the Annotation bodies with the primitive object. // Mirror DEER's client-side expand() (deer-utils.js buildValueObject) diff --git a/controllers/utils.js b/controllers/utils.js index dd455d05..72e2614a 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -109,6 +109,69 @@ const generateSlugId = async function(slug_id="", next){ return slug_return } +// RERUM has minted these two under both 'http' and 'https' over the years, so a filter on either +// must match both spellings. Every other supplied filter key is applied exactly as given. +const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) + +/** + * Find the current (leaf) Annotations targeting an entity, for expansion. + * + * Anticipates the likely Annotation target formats + * - target: 'uri' + * - target: {'id':'uri'} + * - target: {'@id':'uri'} + * and the likely Annotation type formats + * - {"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"} + * + * @param targetId The '@id' or 'id' URI of the entity being expanded. + * @param filters Literal MongoDB filter keys to AND into the query. Already sanitized by the + * caller -- the leaf, type, and target constraints here cannot be overruled. + * @param pagination A {limit, skip} pair from getPagination(). When supplied, the query is sorted + * by '_id' first -- Mongo's natural order is not stable across paged calls, so + * without a sort a client walking pages could miss or repeat Annotations. + * Omit it to fetch every match, which is the long standing expand() behavior. + * @return An Array of matching Annotation documents, with '_id' removed. + */ +const findLeafAnnotationsFor = async function (targetId, filters = {}, pagination = null) { + // '$and' is always present so the filter conditions below can push into it from either branch. + // 'annoTypeConditions' is always pushed, so it is never the empty Array Mongo rejects. + const queryObj = { + "__rerum.history.next": { $exists: true, $size: 0 }, + "$and": [] + } + const annoTypeConditions = [{"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"}] + if (targetId.startsWith("http")) { + const targetConditions = [] + for (const targetKey of ["target", "target.@id", "target.id"]) { + targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) + targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) + } + queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) + } + else { + queryObj["$and"].push({"$or": annoTypeConditions}) + queryObj.target = targetId + } + for (const [key, value] of Object.entries(filters)) { + if (URI_DOUBLED_FILTER_KEYS.has(key) && typeof value === "string" && /^https?:\/\//.test(value)) { + queryObj["$and"].push({"$or": [ + { [key]: value.replace(/^https?/, "http") }, + { [key]: value.replace(/^https?/, "https") } + ]}) + continue + } + queryObj["$and"].push({ [key]: value }) + } + // Get the Annotations targeting this Entity from the db. Remove _id property. + let cursor = db.find(queryObj) + if (pagination) cursor = cursor.sort({ "_id": 1 }).limit(pagination.limit).skip(pagination.skip) + const matches = await cursor.toArray() + return matches.map(o => { + delete o._id + return o + }) +} + // Handle index actions const index = function (req, res, next) { res.json({ @@ -465,6 +528,7 @@ async function healReleasesTree(releasing) { export { _contextid, idNegotiation, + findLeafAnnotationsFor, getPagination, generateSlugId, index, diff --git a/database/__mocks__/index.js b/database/__mocks__/index.js index 51ec6a2c..b155179a 100644 --- a/database/__mocks__/index.js +++ b/database/__mocks__/index.js @@ -41,6 +41,7 @@ function createMockFunction(implementation = () => undefined) { function createCursor() { return { + sort: createMockFunction(function () { return this }), limit: createMockFunction(function () { return this }), skip: createMockFunction(function () { return this }), toArray: createMockFunction(() => Promise.resolve([])) diff --git a/db-controller.js b/db-controller.js index 7f161667..99f5c163 100644 --- a/db-controller.js +++ b/db-controller.js @@ -8,7 +8,7 @@ // Import controller modules import { index, idNegotiation, generateSlugId, remove } from './controllers/utils.js' -import { create, query, id } from './controllers/crud.js' +import { create, query, id, idExpanded } from './controllers/crud.js' import { searchAsWords, searchAsPhrase } from './controllers/search.js' import { deleteObj } from './controllers/delete.js' import { putUpdate, patchUpdate, patchSet, patchUnset, overwrite } from './controllers/update.js' @@ -32,6 +32,7 @@ export default { searchAsWords, searchAsPhrase, id, + idExpanded, bulkCreate, bulkUpdate, queryHeadRequest, diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index 7174b34a..fa603009 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -44,6 +44,99 @@ paths: description: Object headers '404': $ref: '#/components/responses/NotFound' + /id/{id}/expanded: + get: + summary: Read object by id with its current Annotations merged in + operationId: getExpandedObjectById + parameters: + - $ref: '#/components/parameters/ObjectId' + - in: query + name: generator + required: false + description: Only expand with Annotations generated by this registered app agent. + schema: + type: string + - in: query + name: creator + required: false + description: Only expand with Annotations attributed to this creator. + schema: + type: string + - in: query + name: limit + required: false + description: Maximum Annotations to gather. Defaults to 200. + schema: + type: integer + - in: query + name: skip + required: false + description: Annotations to skip before gathering. Defaults to 0. + schema: + type: integer + responses: + '200': + description: Expanded object payload + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + '404': + $ref: '#/components/responses/NotFound' + head: + summary: Read expanded object headers by id + operationId: headExpandedObjectById + parameters: + - $ref: '#/components/parameters/ObjectId' + responses: + '200': + description: Expanded object headers + '404': + $ref: '#/components/responses/NotFound' + post: + summary: Read object by id with its current Annotations merged in, filtered by the request body + operationId: postExpandedObjectById + description: >- + The request body is an object of literal MongoDB filter properties ANDed into the search for + Annotations targeting the entity. URL query parameters supply no filters, though 'limit' and + 'skip' still page the search. The leaf version, the Annotation type, and the target + constraints are applied automatically and cannot be overruled, so 'target', 'type', '@type', + and '__rerum.history' keys are ignored. + parameters: + - $ref: '#/components/parameters/ObjectId' + - in: query + name: limit + required: false + description: Maximum Annotations to gather. Defaults to 200. + schema: + type: integer + - in: query + name: skip + required: false + description: Annotations to skip before gathering. Defaults to 0. + schema: + type: integer + requestBody: + required: false + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + responses: + '200': + description: Expanded object payload + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + '400': + $ref: '#/components/responses/BadRequest' + '404': + $ref: '#/components/responses/NotFound' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '415': + $ref: '#/components/responses/UnsupportedMediaType' /since/{id}: get: summary: Read updates since id diff --git a/public/API.html b/public/API.html index 38f2df93..a105fc3c 100644 --- a/public/API.html +++ b/public/API.html @@ -48,6 +48,7 @@

API (1.1.0)

  • GET @@ -61,6 +62,7 @@

    API (1.1.0)

  • Custom Query
  • Text Search
  • Phrase Search
    • +
  • Expanded record with filters
  • HTTP POST Method Override
    • @@ -149,6 +151,99 @@

    Single record by id

    This can be used directly in the browser. Try it to see what the response resp looks like. https://devstore.rerum.io/v1/id/11111

    +

    Expanded record by id

    + + + + + + + + + + + + + + + +
    PatternPayloadResponse
    /id/_id/expandedempty200 {JSON}
    +

    + Gather every current Annotation targeting the record and merge what those Annotations assert onto it. + This does the entity assembly that client applications otherwise perform with many + /query requests. +

    + +

    + Only the current (leaf) versions of Annotations are gathered. No token is required. + The response is a raw assembly—if your application needs the data in a particular shape, format the + response for your own internal needs. +

    +
    What gets merged
    + +

    +

    Javascript Example
    + 
    
+                const expanded = await fetch("https://devstore.rerum.io/v1/id/11111/expanded").then(resp => resp.json()).catch(err => {throw err})
+
    +

    +

    + This can be used directly in the browser. Try it to see what the response resp looks like. + https://devstore.rerum.io/v1/id/11111/expanded +

    History tree before this version

    @@ -748,6 +843,97 @@ Results are returned sorted by relevance score in descending order. The __rerum.score property indicates match quality.

    +

    Expanded record with filters

    +
    + + + + + + + + + + + + + + +
    PatternPayloadResponse
    /id/_id/expanded{JSON}200 {JSON}
    +

    + The same expansion as + GET /id/_id/expanded, + with the search for Annotations narrowed by the request body. Use this when the convenience parameters on + the GET are not enough. Unlike the GET, this response is not browser cached. +

    + +

    + Three constraints belong to the endpoint and cannot be overruled. Supplying them is not an error—they are + ignored, by exact name or as a dotted prefix. +

    + + + + + + + + + + + + + + + + + + + + + +
    IgnoredAlways applied instead
    target, + target.@id, + target.idAnnotations targeting the record at + _id
    type, + @typeAnnotations only
    __rerum.history.next, + __rerum.history.previous, + __rerum.history.primeCurrent (leaf) versions only
    +

    +

    Javascript Example
    + 
    
+                const expanded = await fetch("https://devstore.rerum.io/v1/id/11111/expanded", {
+                    method: "POST",
+                    headers: { "Content-Type": "application/json" },
+                    body: JSON.stringify({
+                        "__rerum.generatedBy": "https://devstore.rerum.io/v1/id/agent7",
+                        "motivation": "describing"
+                    })
+                }).then(resp => resp.json()).catch(err => {throw err})
+
    +

    HTTP POST Method Override

    This section is non-normative.

    diff --git a/routes/id.js b/routes/id.js index fdfca44f..3e2069a1 100644 --- a/routes/id.js +++ b/routes/id.js @@ -2,6 +2,15 @@ import express from 'express' const router = express.Router() //This controller will handle all MongoDB interactions. import controller from '../db-controller.js' +import rest from '../rest.js' + +router.route('/:_id/expanded') + .get(controller.idExpanded) + .post(rest.verifyJsonContentType, controller.idExpanded) + .all((req, res, next) => { + res.statusMessage = 'Improper request method, please use GET or POST.' + res.status(405).end() + }) router.route('/:_id') .get(controller.id) From c060e50abcdf433e8a9d72a3a1d47b6434ad5fb7 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Fri, 7 Aug 2026 13:40:52 -0500 Subject: [PATCH 02/48] Catch the W3C SpecificResource form target variants as well --- controllers/utils.js | 6 +++++- public/API.html | 14 +++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/controllers/utils.js b/controllers/utils.js index 72e2614a..8aeb499a 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -120,6 +120,8 @@ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) * - target: 'uri' * - target: {'id':'uri'} * - target: {'@id':'uri'} + * - target: {'source':'uri', 'type':'SpecificResource'} the W3C SpecificResource + * - target: {'source':{'id':'uri'}} a SpecificResource with an embedded source * and the likely Annotation type formats * - {"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"} * @@ -142,7 +144,9 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}, paginatio const annoTypeConditions = [{"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"}] if (targetId.startsWith("http")) { const targetConditions = [] - for (const targetKey of ["target", "target.@id", "target.id"]) { + // 'target.source' is the W3C SpecificResource, which is how an Annotation targets a + // fragment or a selected region of a resource rather than the whole of it. + for (const targetKey of ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"]) { targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) } diff --git a/public/API.html b/public/API.html index a105fc3c..30c56868 100644 --- a/public/API.html +++ b/public/API.html @@ -225,7 +225,10 @@

    Expanded record by id

    _id, __rerum, __deleted, and - @context. + @context. An assertion naming + __proto__ is dropped for the same reason—it + is not data, and emitting it would hand a prototype pollution vector to every client that parses the + response.
  • Skipped for now, as future work: a body with more than one property, an Annotation with multiple bodies, the Choice, @@ -904,9 +907,14 @@

    Expanded record with filters

    target, target.@id, - target.id + target.id, + target.source, + target.source.@id, + target.source.id Annotations targeting the record at - _id + _id, including those + targeting it through a + SpecificResource type, From 1cb50865b180c21dbf47656bcb783921c897fa6b Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 11 Aug 2026 10:44:32 -0500 Subject: [PATCH 03/48] changes during review --- controllers/crud.js | 19 +++++++++++++++++-- controllers/gog.js | 13 ++++++++++--- controllers/utils.js | 24 ++++++++++++++++++++++-- public/API.html | 4 +++- 4 files changed, 52 insertions(+), 8 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index bcadc537..5c7946f3 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -159,6 +159,14 @@ function sanitizeExpansionFilters(supplied) { return filters } +/** + * The Annotation body types whose value is kept whole rather than read as a single assertion. + * The OA prefixed spelling is honored for the Annotation type in findLeafAnnotationsFor(), so it + * is honored here too. Without it an 'oa:TextualBody' falls through to the single key check and + * is dropped, since a TextualBody always carries at least a type and a value. + */ +const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody"]) + /** * The [key, value] assertions an Annotation makes about the entity it targets. * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other @@ -169,6 +177,7 @@ function sanitizeExpansionFilters(supplied) { * - body: {'key': 'value'} a single assertion * - body: {'key': {...}} a single assertion, value kept as-is * - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive + * - body: {'@type':'oa:TextualBody', ...} the OA prefixed spelling of the same * * @param anno An Annotation document. * @return An Array of [key, value] pairs to merge onto the entity. @@ -180,7 +189,7 @@ function assertionsFrom(anno) { // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an // external resource with no embedded value to expand with. if (!body || typeof body !== "object" || Array.isArray(body)) return assertions - if ((body.type ?? body["@type"]) === "TextualBody") { + if (TEXTUAL_BODY_TYPES.has(body.type ?? body["@type"])) { assertions.push(["bodyValue", body]) return assertions } @@ -280,9 +289,15 @@ const idExpanded = async function (req, res, next) { // Let clients detect a full page. When this equals the limit there may be more to gather, // and the entity in hand is expanded from only part of its Annotations. res.set('Annotations-Merged', String(annos.length)) + // This deployment's '/expanded' URI, not the entity URI. The entity URI would hand back + // the unexpanded record, and it cannot be the base for this one either -- an entity minted + // by another RERUM carries that host in its stored '@id', and there is no guarantee the + // other host serves '/expanded' at all. RERUM_ID_PREFIX is how idNegotiation() mints ids, + // so this stays on the host actually answering the request. + const expandedLocation = `${process.env.RERUM_ID_PREFIX}${match._id}/expanded` let expanded = applyRawExpansion(match, annos) expanded = idNegotiation(expanded) - res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) + res.location(expandedLocation) res.json(expanded) } catch (error) { return next(utils.createExpressError(error)) diff --git a/controllers/gog.js b/controllers/gog.js index bfbe1b7c..e575910d 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -8,7 +8,7 @@ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor } from './utils.js' +import { ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor } from './utils.js' // The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different // agents; only the trailing id is compared, so either host spelling matches. @@ -364,7 +364,9 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde }, evidence: assertion?.evidence ?? anno.evidence ?? "" } - if(expandedEntity.hasOwnProperty(key)){ + // Object.hasOwn() rather than the method on the entity. A merged assertion named + // 'hasOwnProperty' would shadow the method and throw a TypeError on the next iteration. + if(Object.hasOwn(expandedEntity, key)){ expandedEntity[key] = Array.isArray(expandedEntity[key]) ? [...expandedEntity[key], valueObject] : [expandedEntity[key], valueObject] @@ -401,6 +403,11 @@ const expandedId = async function (req, res, next) { }) return next(utils.createExpressError(err)) } + // This '/gog/id' URI, not the entity URI. This response is the expanded representation, + // and the entity URI would hand back the unexpanded record instead. Built off + // RERUM_ID_PREFIX so the origin follows the deployment, and captured before expand() in + // case idNegotiation() reaches the match itself and drops '_id'. + const expandedLocation = new URL(`/gog/id/${match._id}`, process.env.RERUM_ID_PREFIX).href // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). res.set(utils.configureWebAnnoHeadersFor(match)) res.set("Cache-Control", "max-age=86400, must-revalidate") @@ -409,7 +416,7 @@ const expandedId = async function (req, res, next) { res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "") let expanded = await expand(match, generator) expanded = idNegotiation(expanded) - res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) + res.location(expandedLocation) res.json(expanded) } catch (error) { return next(utils.createExpressError(error)) diff --git a/controllers/utils.js b/controllers/utils.js index 8aeb499a..4fbf0f2a 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -113,6 +113,19 @@ const generateSlugId = async function(slug_id="", next){ // must match both spellings. Every other supplied filter key is applied exactly as given. const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) +// The properties an Annotation can carry the URI of its target under. +const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"] + +/** + * Escape the RegExp metacharacters in a literal so it can be embedded in a pattern and match only + * itself. A RERUM URI has at least the dots of its host to escape. + * @param literal A string to be matched literally. + * @return The same string, safe to concatenate into a RegExp source. + */ +function escapeRegex(literal) { + return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +} + /** * Find the current (leaf) Annotations targeting an entity, for expansion. * @@ -122,6 +135,7 @@ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) * - target: {'@id':'uri'} * - target: {'source':'uri', 'type':'SpecificResource'} the W3C SpecificResource * - target: {'source':{'id':'uri'}} a SpecificResource with an embedded source + * - target: 'uri#xywh=0,0,100,100' a fragment of the resource * and the likely Annotation type formats * - {"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"} * @@ -144,11 +158,17 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}, paginatio const annoTypeConditions = [{"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"}] if (targetId.startsWith("http")) { const targetConditions = [] + // Hanging a fragment off the URI is the other W3C way to target part of a resource rather + // than the whole of it, and an exact match will not catch one. Anchored at the front so + // the pattern can still use an index, and terminated by the '#' so it cannot spill onto a + // longer id. One pattern covers both spellings, since only the scheme is left unescaped. + const fragmentPattern = new RegExp(`^https?${escapeRegex(targetId.replace(/^https?/, ""))}#`) // 'target.source' is the W3C SpecificResource, which is how an Annotation targets a - // fragment or a selected region of a resource rather than the whole of it. - for (const targetKey of ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"]) { + // selected region of a resource rather than the whole of it. + for (const targetKey of TARGET_KEYS) { targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) + targetConditions.push({ [targetKey]: fragmentPattern }) } queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) } diff --git a/public/API.html b/public/API.html index 30c56868..45838f66 100644 --- a/public/API.html +++ b/public/API.html @@ -914,7 +914,9 @@

    Expanded record with filters

    Annotations targeting the record at _id, including those targeting it through a - SpecificResource + SpecificResource or through a + fragment of its URI such as + #xywh=0,0,100,100 type, From e39ff2e5d684bb2b7d6957330deb4e38c4c4e3f2 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 11 Aug 2026 11:21:55 -0500 Subject: [PATCH 04/48] changes during review --- controllers/gog.js | 7 ++++++- controllers/utils.js | 21 ++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/controllers/gog.js b/controllers/gog.js index e575910d..166c02f4 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -323,6 +323,7 @@ const _gog_glosses_from_manuscript = async function (req, res, next) { * * Anticipate likely Annotation type formats * - {"type": "Annotation"} +* - {"type": "oa:Annotation"} * - {"@type": "Annotation"} * - {"@type": "oa:Annotation"} * @@ -350,7 +351,11 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde let expandedEntity = structuredClone(primitiveEntity) for(const anno of matches){ const body = anno.body - if(!body || typeof body !== "object") continue + // Array.isArray() as well as the typeof check. An Array is a typeof 'object', and + // Object.keys() on a one element Array is ["0"] -- a length of 1 that would pass the + // single assertion check below and merge the body onto the entity under the key "0". + // Annotations carrying multiple bodies are not expanded with. + if(!body || typeof body !== "object" || Array.isArray(body)) continue const keys = Object.keys(body) if(keys.length !== 1) continue const key = keys[0] diff --git a/controllers/utils.js b/controllers/utils.js index 4fbf0f2a..a4e7a793 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -137,7 +137,8 @@ function escapeRegex(literal) { * - target: {'source':{'id':'uri'}} a SpecificResource with an embedded source * - target: 'uri#xywh=0,0,100,100' a fragment of the resource * and the likely Annotation type formats - * - {"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"} + * - {"type": "Annotation"}, {"type": "oa:Annotation"} + * - {"@type": "Annotation"}, {"@type": "oa:Annotation"} * * @param targetId The '@id' or 'id' URI of the entity being expanded. * @param filters Literal MongoDB filter keys to AND into the query. Already sanitized by the @@ -155,20 +156,26 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}, paginatio "__rerum.history.next": { $exists: true, $size: 0 }, "$and": [] } - const annoTypeConditions = [{"type": "Annotation"}, {"@type": "Annotation"}, {"@type": "oa:Annotation"}] + const annoTypeConditions = [ + {"type": "Annotation"}, {"type": "oa:Annotation"}, + {"@type": "Annotation"}, {"@type": "oa:Annotation"} + ] if (targetId.startsWith("http")) { const targetConditions = [] // Hanging a fragment off the URI is the other W3C way to target part of a resource rather - // than the whole of it, and an exact match will not catch one. Anchored at the front so - // the pattern can still use an index, and terminated by the '#' so it cannot spill onto a - // longer id. One pattern covers both spellings, since only the scheme is left unescaped. - const fragmentPattern = new RegExp(`^https?${escapeRegex(targetId.replace(/^https?/, ""))}#`) + // than the whole of it, and an exact match will not catch one. Anchored at the front and + // terminated by the '#' so the pattern cannot spill onto a longer id. One pattern per + // scheme rather than a single '^https?' -- Mongo bounds an index scan by the pattern's + // literal prefix, and '^https?' leaves it only 'http', which is every target URI stored. + const fragmentPatterns = ["http", "https"].map(scheme => + new RegExp(`^${escapeRegex(targetId.replace(/^https?/, scheme))}#`) + ) // 'target.source' is the W3C SpecificResource, which is how an Annotation targets a // selected region of a resource rather than the whole of it. for (const targetKey of TARGET_KEYS) { targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) - targetConditions.push({ [targetKey]: fragmentPattern }) + for (const fragmentPattern of fragmentPatterns) targetConditions.push({ [targetKey]: fragmentPattern }) } queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) } From 780528a0a7c66ae46ed4b0a00b657286430260ca Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Mon, 17 Aug 2026 09:23:59 -0500 Subject: [PATCH 05/48] changes from review and testing --- controllers/crud.js | 28 +++++++++++++++++++++++----- controllers/utils.js | 18 +++++++++++++----- public/API.html | 8 +++++--- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index 5c7946f3..b3968766 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -167,6 +167,18 @@ function sanitizeExpansionFilters(supplied) { */ const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody"]) +/** + * Whether an Annotation carries the W3C multiple bodies form, which is an Array of bodies. + * That form is future work, so such an Annotation asserts nothing an expansion can use. Both + * assertionsFrom() and the 'Annotations-Gathered' count read this, so the two cannot disagree + * about what does and does not contribute. + * @param anno An Annotation document. + * @return A boolean, true when the Annotation carries more than one body. + */ +function hasMultipleBodies(anno) { + return Array.isArray(anno.body) +} + /** * The [key, value] assertions an Annotation makes about the entity it targets. * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other @@ -188,7 +200,7 @@ function assertionsFrom(anno) { const body = anno.body // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an // external resource with no embedded value to expand with. - if (!body || typeof body !== "object" || Array.isArray(body)) return assertions + if (hasMultipleBodies(anno) || !body || typeof body !== "object") return assertions if (TEXTUAL_BODY_TYPES.has(body.type ?? body["@type"])) { assertions.push(["bodyValue", body]) return assertions @@ -287,14 +299,20 @@ const idExpanded = async function (req, res, next) { const targetId = match["@id"] ?? match.id const annos = targetId ? await findLeafAnnotationsFor(targetId, filters, pagination) : [] // Let clients detect a full page. When this equals the limit there may be more to gather, - // and the entity in hand is expanded from only part of its Annotations. - res.set('Annotations-Merged', String(annos.length)) + // and the entity in hand is expanded from only part of its Annotations. This is the count + // gathered, not the count that changed the entity -- a gathered Annotation still asserts + // nothing when its body is protected or structural. Annotations carrying multiple bodies + // are the exception and are left out, since that form is not read at all yet and counting + // them would claim work this endpoint has not been built to do. + const gathered = annos.filter(anno => !hasMultipleBodies(anno)) + res.set('Annotations-Gathered', String(gathered.length)) // This deployment's '/expanded' URI, not the entity URI. The entity URI would hand back // the unexpanded record, and it cannot be the base for this one either -- an entity minted // by another RERUM carries that host in its stored '@id', and there is no guarantee the // other host serves '/expanded' at all. RERUM_ID_PREFIX is how idNegotiation() mints ids, - // so this stays on the host actually answering the request. - const expandedLocation = `${process.env.RERUM_ID_PREFIX}${match._id}/expanded` + // so this stays on the host actually answering the request. Resolved through 'new URL()', + // the same way gog.js builds its expanded Location. + const expandedLocation = new URL(`${match._id}/expanded`, process.env.RERUM_ID_PREFIX).href let expanded = applyRawExpansion(match, annos) expanded = idNegotiation(expanded) res.location(expandedLocation) diff --git a/controllers/utils.js b/controllers/utils.js index a4e7a793..f4df737a 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -116,6 +116,10 @@ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) // The properties an Annotation can carry the URI of its target under. const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"] +// The JSON-LD expanded form of the Annotation type, alongside the compact 'Annotation' and the +// 'oa:' prefixed spellings. +const OA_ANNOTATION_IRI = "http://www.w3.org/ns/oa#Annotation" + /** * Escape the RegExp metacharacters in a literal so it can be embedded in a pattern and match only * itself. A RERUM URI has at least the dots of its host to escape. @@ -137,8 +141,8 @@ function escapeRegex(literal) { * - target: {'source':{'id':'uri'}} a SpecificResource with an embedded source * - target: 'uri#xywh=0,0,100,100' a fragment of the resource * and the likely Annotation type formats - * - {"type": "Annotation"}, {"type": "oa:Annotation"} - * - {"@type": "Annotation"}, {"@type": "oa:Annotation"} + * - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"} + * - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"} * * @param targetId The '@id' or 'id' URI of the entity being expanded. * @param filters Literal MongoDB filter keys to AND into the query. Already sanitized by the @@ -156,11 +160,15 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}, paginatio "__rerum.history.next": { $exists: true, $size: 0 }, "$and": [] } + // The compact, the 'oa:' prefixed, and the JSON-LD expanded IRI spellings of the Annotation + // type. All three are valid ways to say the same thing, and RERUM stores whatever it is given. const annoTypeConditions = [ - {"type": "Annotation"}, {"type": "oa:Annotation"}, - {"@type": "Annotation"}, {"@type": "oa:Annotation"} + {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": OA_ANNOTATION_IRI}, + {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": OA_ANNOTATION_IRI} ] - if (targetId.startsWith("http")) { + // The same scheme test the filter values below use. 'startsWith("http")' would also pass a URI + // like 'httpx://host/thing', which the scheme swap would then mangle into 'httpsx://host/thing'. + if (/^https?:\/\//.test(targetId)) { const targetConditions = [] // Hanging a fragment off the URI is the other W3C way to target part of a resource rather // than the whole of it, and an exact match will not catch one. Anchored at the front and diff --git a/public/API.html b/public/API.html index 45838f66..fdb78483 100644 --- a/public/API.html +++ b/public/API.html @@ -189,8 +189,10 @@

    Expanded record by id

    with identifier _id, plus the merged properties.
  • Response header - Annotations-Merged—how many - Annotations went into this expansion. When it equals your + Annotations-Gathered—how many + Annotations were gathered for this expansion. A gathered Annotation may still assert nothing + mergeable, so this is not a count of the properties you received. Annotations with multiple bodies + are not counted at all, as that form is not read yet. When it equals your limit there may be more, and the record you received was expanded from only part of its Annotations. Raise the limit or walk pages with @@ -881,7 +883,7 @@

    Expanded record with filters

    here—supply them in the body instead. Paging is not a filter, so ?limit and ?skip still work, and the - Annotations-Merged response header is set + Annotations-Gathered response header is set exactly as it is on the GET.
  • __rerum.generatedBy and creator are matched against both the From 967ee7fa8d9c5e7ceaedcb4cbc53aedf8044a682 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Mon, 17 Aug 2026 11:51:35 -0500 Subject: [PATCH 06/48] changes from testing and review --- controllers/crud.js | 24 ++++++++++++++---------- controllers/utils.js | 1 + public/API.html | 23 +++++++++++++++-------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index b3968766..9c21252a 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -165,12 +165,12 @@ function sanitizeExpansionFilters(supplied) { * is honored here too. Without it an 'oa:TextualBody' falls through to the single key check and * is dropped, since a TextualBody always carries at least a type and a value. */ -const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody"]) +const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www.w3.org/ns/oa#TextualBody"]) /** * Whether an Annotation carries the W3C multiple bodies form, which is an Array of bodies. * That form is future work, so such an Annotation asserts nothing an expansion can use. Both - * assertionsFrom() and the 'Annotations-Gathered' count read this, so the two cannot disagree + * assertionsFrom() and the 'Annotations-Merged' count read this, so the two cannot disagree * about what does and does not contribute. * @param anno An Annotation document. * @return A boolean, true when the Annotation carries more than one body. @@ -299,20 +299,24 @@ const idExpanded = async function (req, res, next) { const targetId = match["@id"] ?? match.id const annos = targetId ? await findLeafAnnotationsFor(targetId, filters, pagination) : [] // Let clients detect a full page. When this equals the limit there may be more to gather, - // and the entity in hand is expanded from only part of its Annotations. This is the count - // gathered, not the count that changed the entity -- a gathered Annotation still asserts - // nothing when its body is protected or structural. Annotations carrying multiple bodies - // are the exception and are left out, since that form is not read at all yet and counting - // them would claim work this endpoint has not been built to do. - const gathered = annos.filter(anno => !hasMultipleBodies(anno)) - res.set('Annotations-Gathered', String(gathered.length)) + // and the entity in hand is expanded from only part of its Annotations. This has to be the + // raw page size -- subtracting the Annotations that assert nothing would let a full page + // read as a partial one and stop a client paging before it has everything. + res.set('Annotations-Gathered', String(annos.length)) + // How many of the gathered Annotations could contribute, which is a different number and + // gets its own header rather than quietly standing in for the one above. This is still not + // a count of the properties received -- a gathered Annotation asserts nothing when its body + // is protected or structural. Annotations carrying multiple bodies are left out here, + // since that form is not read at all yet. + const merged = annos.filter(anno => !hasMultipleBodies(anno)) + res.set('Annotations-Merged', String(merged.length)) // This deployment's '/expanded' URI, not the entity URI. The entity URI would hand back // the unexpanded record, and it cannot be the base for this one either -- an entity minted // by another RERUM carries that host in its stored '@id', and there is no guarantee the // other host serves '/expanded' at all. RERUM_ID_PREFIX is how idNegotiation() mints ids, // so this stays on the host actually answering the request. Resolved through 'new URL()', // the same way gog.js builds its expanded Location. - const expandedLocation = new URL(`${match._id}/expanded`, process.env.RERUM_ID_PREFIX).href + const expandedLocation = new URL(`/v1/id/${match._id}/expanded`, process.env.RERUM_ID_PREFIX).href let expanded = applyRawExpansion(match, annos) expanded = idNegotiation(expanded) res.location(expandedLocation) diff --git a/controllers/utils.js b/controllers/utils.js index f4df737a..e5886e7e 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -114,6 +114,7 @@ const generateSlugId = async function(slug_id="", next){ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) // The properties an Annotation can carry the URI of its target under. +// Choice, Composite, and List target constructs, whose members sit in an 'items' Array. They are not supported here. const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"] // The JSON-LD expanded form of the Annotation type, alongside the compact 'Annotation' and the diff --git a/public/API.html b/public/API.html index fdb78483..e1b95e30 100644 --- a/public/API.html +++ b/public/API.html @@ -148,7 +148,7 @@

    Single record by id

    - This can be used directly in the browser. Try it to see what the response resp looks like. + This can be used directly in the browser. Try it to see what the response resp looks like. https://devstore.rerum.io/v1/id/11111

    Expanded record by id

    @@ -190,13 +190,19 @@

    Expanded record by id

    properties.
  • Response header Annotations-Gathered—how many - Annotations were gathered for this expansion. A gathered Annotation may still assert nothing - mergeable, so this is not a count of the properties you received. Annotations with multiple bodies - are not counted at all, as that form is not read yet. When it equals your + Annotations this expansion gathered, which is the size of the page you received. When it equals your limit there may be more, and the record you received was expanded from only part of its Annotations. Raise the limit or walk pages with skip.
    • +
  • Response header + Annotations-Merged—how many of + those gathered Annotations could contribute to the record. It is always less than or equal to + Annotations-Gathered, and Annotations with + multiple bodies are not counted, as that form is not read yet. A counted Annotation may still assert + nothing mergeable, so this is not a count of the properties you received either. Use + Annotations-Gathered, not this one, to decide + whether to keep paging.

    • Only the current (leaf) versions of Annotations are gathered. No token is required. @@ -246,7 +252,7 @@

    Expanded record by id

    - This can be used directly in the browser. Try it to see what the response resp looks like. + This can be used directly in the browser. Try it to see what the response resp looks like. https://devstore.rerum.io/v1/id/11111/expanded

    History tree before this version

    @@ -285,7 +291,7 @@

    History tree before this version

    - This can be used directly in the browser. Try it to see what the response resp looks like. + This can be used directly in the browser. Try it to see what the response resp looks like. https://devstore.rerum.io/v1/history/11111

    History tree since this version

    @@ -883,8 +889,9 @@

    Expanded record with filters

    here—supply them in the body instead. Paging is not a filter, so ?limit and ?skip still work, and the - Annotations-Gathered response header is set - exactly as it is on the GET. + Annotations-Gathered and + Annotations-Merged response headers are set + exactly as they are on the GET.
  • __rerum.generatedBy and creator are matched against both the http and From 79dc5bd3d36fe15dd8fd7c9b5cbdfacec1b850fe Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Mon, 17 Aug 2026 13:41:27 -0500 Subject: [PATCH 07/48] comment cleanup --- controllers/crud.js | 64 ++++++++------------------------------------ controllers/gog.js | 9 ------- controllers/utils.js | 38 +++++++++----------------- 3 files changed, 24 insertions(+), 87 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index 9c21252a..c108de83 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -131,17 +131,12 @@ const id = async function (req, res, next) { * The expand job always constrains the Annotations it gathers to the leaf versions, to the * Annotation types, and to the entity in the request URI. A client cannot influence those, so * these keys are dropped from a supplied filter body by exact name or dotted prefix. - * The dot matters -- 'targetCollection' is a real property on Gallery of Glosses data and must - * still be usable as a filter. */ const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"] /** * Identity and system properties an Annotation body must never overwrite. '@id' and '@context' - * are read by idNegotiation() and res.location() right after the merge, so clobbering them would - * break the response itself. '__proto__' is not data -- assigning it would re-point the response - * object's prototype instead of adding a property, and emitting it would hand a prototype - * pollution vector to every client that parses the response. + * are read by idNegotiation() and res.location() */ const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"]) @@ -161,24 +156,9 @@ function sanitizeExpansionFilters(supplied) { /** * The Annotation body types whose value is kept whole rather than read as a single assertion. - * The OA prefixed spelling is honored for the Annotation type in findLeafAnnotationsFor(), so it - * is honored here too. Without it an 'oa:TextualBody' falls through to the single key check and - * is dropped, since a TextualBody always carries at least a type and a value. */ const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www.w3.org/ns/oa#TextualBody"]) -/** - * Whether an Annotation carries the W3C multiple bodies form, which is an Array of bodies. - * That form is future work, so such an Annotation asserts nothing an expansion can use. Both - * assertionsFrom() and the 'Annotations-Merged' count read this, so the two cannot disagree - * about what does and does not contribute. - * @param anno An Annotation document. - * @return A boolean, true when the Annotation carries more than one body. - */ -function hasMultipleBodies(anno) { - return Array.isArray(anno.body) -} - /** * The [key, value] assertions an Annotation makes about the entity it targets. * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other @@ -200,15 +180,14 @@ function assertionsFrom(anno) { const body = anno.body // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an // external resource with no embedded value to expand with. - if (hasMultipleBodies(anno) || !body || typeof body !== "object") return assertions + if (Array.isArray(body) || !body || typeof body !== "object") return assertions if (TEXTUAL_BODY_TYPES.has(body.type ?? body["@type"])) { assertions.push(["bodyValue", body]) return assertions } const keys = Object.keys(body) // Any other multi-key body is structural rather than assertional and cannot be attributed to a - // single entity property. This is what skips the Choice, Composite, and List multiplicity - // constructs, which are all shaped {type, items}. + // single entity property. This is what skips the Choice, Composite, and List multiplicity constructs. if (keys.length !== 1) return assertions assertions.push([keys[0], body[keys[0]]]) return assertions @@ -216,17 +195,15 @@ function assertionsFrom(anno) { /** * Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values. - * Unlike the Gallery of Glosses expand(), values are not wrapped and not unwrapped -- what the - * Annotation says is what the entity gets. When more than one current Annotation asserts the same - * key, or the entity already carries it, the values collect into an Array. + * When more than one current Annotation asserts the same key, or the entity already carries it, + * the values collect into an Array. * @param primitiveEntity The unexpanded entity. * @param annos The Annotations targeting it. * @return A new, expanded entity object. */ function applyRawExpansion(primitiveEntity, annos) { const expandedEntity = structuredClone(primitiveEntity) - // Hold __rerum aside so it can be re-appended after the merged properties. It is the - // last property on a stored object and should stay last on an expanded one. + // Hold __rerum aside so it can be re-appended after the merged properties. It will be the last property. const rerumProp = expandedEntity.__rerum delete expandedEntity.__rerum for (const anno of annos) { @@ -246,7 +223,7 @@ function applyRawExpansion(primitiveEntity, annos) { /** * Query the MongoDB for the object with the _id or __rerum.slug provided in the request URL, then - * merge in the assertions of all the current Annotations targeting it. + * merge in the assertions of all the current leaf Annotations targeting it. * * GET recognizes the '?generator=' and '?creator=' convenience parameters only. * POST reads literal MongoDB filter keys from the JSON body and ignores URL parameters as filters. @@ -256,9 +233,6 @@ const idExpanded = async function (req, res, next) { res.set("Content-Type", "application/json; charset=utf-8") const id = req.params["_id"] const isPost = req.method === "POST" - //Paging is transport rather than a filter, so it comes off the URL for both methods. - //The default is generous because an expansion wants every Annotation it can get, and an - //entity with more than 200 targeting it is not expected. const pagination = getPagination(req.query, 200) let filters = {} if (isPost) { @@ -290,32 +264,16 @@ const idExpanded = async function (req, res, next) { res.set(utils.configureWebAnnoHeadersFor(match)) //Support built in browser caching. A POST response is not cacheable. if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate") - // No Last-Modified here, unlike GET /v1/id/:_id. It would compare against the root entity - // before the targeting Annotations are merged in. // Include current version for optimistic locking res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "") - // Annotations target the stored URI, so this must come off the raw match. idNegotiation() - // below rebuilds 'id' from RERUM_ID_PREFIX, which is not necessarily the stored host. const targetId = match["@id"] ?? match.id const annos = targetId ? await findLeafAnnotationsFor(targetId, filters, pagination) : [] - // Let clients detect a full page. When this equals the limit there may be more to gather, - // and the entity in hand is expanded from only part of its Annotations. This has to be the - // raw page size -- subtracting the Annotations that assert nothing would let a full page - // read as a partial one and stop a client paging before it has everything. + // Let clients detect a full page. When this equals the limit there may be more to gather. res.set('Annotations-Gathered', String(annos.length)) - // How many of the gathered Annotations could contribute, which is a different number and - // gets its own header rather than quietly standing in for the one above. This is still not - // a count of the properties received -- a gathered Annotation asserts nothing when its body - // is protected or structural. Annotations carrying multiple bodies are left out here, - // since that form is not read at all yet. - const merged = annos.filter(anno => !hasMultipleBodies(anno)) + // How many of the gathered Annotations could contribute, which is a different number. + // Annotations carrying multiple bodies are left out here + const merged = annos.filter(anno => !Array.isArray(anno.body)) res.set('Annotations-Merged', String(merged.length)) - // This deployment's '/expanded' URI, not the entity URI. The entity URI would hand back - // the unexpanded record, and it cannot be the base for this one either -- an entity minted - // by another RERUM carries that host in its stored '@id', and there is no guarantee the - // other host serves '/expanded' at all. RERUM_ID_PREFIX is how idNegotiation() mints ids, - // so this stays on the host actually answering the request. Resolved through 'new URL()', - // the same way gog.js builds its expanded Location. const expandedLocation = new URL(`/v1/id/${match._id}/expanded`, process.env.RERUM_ID_PREFIX).href let expanded = applyRawExpansion(match, annos) expanded = idNegotiation(expanded) diff --git a/controllers/gog.js b/controllers/gog.js index 166c02f4..db011660 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -351,9 +351,6 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde let expandedEntity = structuredClone(primitiveEntity) for(const anno of matches){ const body = anno.body - // Array.isArray() as well as the typeof check. An Array is a typeof 'object', and - // Object.keys() on a one element Array is ["0"] -- a length of 1 that would pass the - // single assertion check below and merge the body onto the entity under the key "0". // Annotations carrying multiple bodies are not expanded with. if(!body || typeof body !== "object" || Array.isArray(body)) continue const keys = Object.keys(body) @@ -369,8 +366,6 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde }, evidence: assertion?.evidence ?? anno.evidence ?? "" } - // Object.hasOwn() rather than the method on the entity. A merged assertion named - // 'hasOwnProperty' would shadow the method and throw a TypeError on the next iteration. if(Object.hasOwn(expandedEntity, key)){ expandedEntity[key] = Array.isArray(expandedEntity[key]) ? [...expandedEntity[key], valueObject] @@ -408,10 +403,6 @@ const expandedId = async function (req, res, next) { }) return next(utils.createExpressError(err)) } - // This '/gog/id' URI, not the entity URI. This response is the expanded representation, - // and the entity URI would hand back the unexpanded record instead. Built off - // RERUM_ID_PREFIX so the origin follows the deployment, and captured before expand() in - // case idNegotiation() reaches the match itself and drops '_id'. const expandedLocation = new URL(`/gog/id/${match._id}`, process.env.RERUM_ID_PREFIX).href // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). res.set(utils.configureWebAnnoHeadersFor(match)) diff --git a/controllers/utils.js b/controllers/utils.js index e5886e7e..d3c7e12a 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -109,18 +109,18 @@ const generateSlugId = async function(slug_id="", next){ return slug_return } -// RERUM has minted these two under both 'http' and 'https' over the years, so a filter on either -// must match both spellings. Every other supplied filter key is applied exactly as given. +/** + * RERUM has minted these two under both 'http' and 'https' over the years, so a filter on either + * must match both spellings. Every other supplied filter key is applied exactly as given. + */ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) -// The properties an Annotation can carry the URI of its target under. -// Choice, Composite, and List target constructs, whose members sit in an 'items' Array. They are not supported here. +/** + * The properties an Annotation can carry the URI of its target under. + * Choice, Composite, and List target constructs, whose members sit in an 'items' Array. They are not supported here. + */ const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"] -// The JSON-LD expanded form of the Annotation type, alongside the compact 'Annotation' and the -// 'oa:' prefixed spellings. -const OA_ANNOTATION_IRI = "http://www.w3.org/ns/oa#Annotation" - /** * Escape the RegExp metacharacters in a literal so it can be embedded in a pattern and match only * itself. A RERUM URI has at least the dots of its host to escape. @@ -146,36 +146,24 @@ function escapeRegex(literal) { * - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"} * * @param targetId The '@id' or 'id' URI of the entity being expanded. - * @param filters Literal MongoDB filter keys to AND into the query. Already sanitized by the - * caller -- the leaf, type, and target constraints here cannot be overruled. - * @param pagination A {limit, skip} pair from getPagination(). When supplied, the query is sorted - * by '_id' first -- Mongo's natural order is not stable across paged calls, so - * without a sort a client walking pages could miss or repeat Annotations. - * Omit it to fetch every match, which is the long standing expand() behavior. + * @param filters Literal MongoDB filter keys to AND into the query. + * @param pagination A {limit, skip} pair from getPagination(). * @return An Array of matching Annotation documents, with '_id' removed. */ const findLeafAnnotationsFor = async function (targetId, filters = {}, pagination = null) { // '$and' is always present so the filter conditions below can push into it from either branch. - // 'annoTypeConditions' is always pushed, so it is never the empty Array Mongo rejects. const queryObj = { "__rerum.history.next": { $exists: true, $size: 0 }, "$and": [] } - // The compact, the 'oa:' prefixed, and the JSON-LD expanded IRI spellings of the Annotation - // type. All three are valid ways to say the same thing, and RERUM stores whatever it is given. const annoTypeConditions = [ - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": OA_ANNOTATION_IRI}, - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": OA_ANNOTATION_IRI} + {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"}, + {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"} ] - // The same scheme test the filter values below use. 'startsWith("http")' would also pass a URI - // like 'httpx://host/thing', which the scheme swap would then mangle into 'httpsx://host/thing'. if (/^https?:\/\//.test(targetId)) { const targetConditions = [] // Hanging a fragment off the URI is the other W3C way to target part of a resource rather - // than the whole of it, and an exact match will not catch one. Anchored at the front and - // terminated by the '#' so the pattern cannot spill onto a longer id. One pattern per - // scheme rather than a single '^https?' -- Mongo bounds an index scan by the pattern's - // literal prefix, and '^https?' leaves it only 'http', which is every target URI stored. + // than the whole of it, and an exact match will not catch one. const fragmentPatterns = ["http", "https"].map(scheme => new RegExp(`^${escapeRegex(targetId.replace(/^https?/, scheme))}#`) ) From 635061b7c66075caddcd5979730e02c51561195f Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Mon, 17 Aug 2026 13:45:07 -0500 Subject: [PATCH 08/48] comment cleanup --- database/__mocks__/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/database/__mocks__/index.js b/database/__mocks__/index.js index b155179a..51ec6a2c 100644 --- a/database/__mocks__/index.js +++ b/database/__mocks__/index.js @@ -41,7 +41,6 @@ function createMockFunction(implementation = () => undefined) { function createCursor() { return { - sort: createMockFunction(function () { return this }), limit: createMockFunction(function () { return this }), skip: createMockFunction(function () { return this }), toArray: createMockFunction(() => Promise.resolve([])) From 749cc51ba15a3c7f9d6c627ffc808c50c3a9c368 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Mon, 17 Aug 2026 15:52:12 -0500 Subject: [PATCH 09/48] cleanup during review --- controllers/gog.js | 12 ++---------- public/API.html | 3 +++ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/controllers/gog.js b/controllers/gog.js index db011660..2eb4ecdf 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -316,16 +316,8 @@ const _gog_glosses_from_manuscript = async function (req, res, next) { * - anno.body * - anno.body.value * -* Anticipate likely Annotation target formats -* - target: 'uri' -* - target: {'id':'uri'} -* - target: {'@id':'uri'} -* -* Anticipate likely Annotation type formats -* - {"type": "Annotation"} -* - {"type": "oa:Annotation"} -* - {"@type": "Annotation"} -* - {"@type": "oa:Annotation"} +* Gathering the Annotations is findLeafAnnotationsFor() in ./utils.js -- see its docblock for the +* target and Annotation type forms that are recognized. This function only reads their bodies. * * @param primitiveEntity - An existing RERUM object * @param GENERATOR - A registered RERUM app's User Agent diff --git a/public/API.html b/public/API.html index e1b95e30..54f3bec3 100644 --- a/public/API.html +++ b/public/API.html @@ -227,6 +227,9 @@

    Expanded record by id

    language survive.
  • When several Annotations assert the same property, or the record already carries it, the values collect into an Array. Array order is not guaranteed.
    • +
  • A property merged from a single Annotation is that value as-is; the same property merged from two or + more is an Array. Treat every merged property as possibly-an-Array—the shape depends on how many + Annotations exist, which changes over the record's life.
  • Identity and system properties are never overwritten: @id, id, From 578fa9e77a3051456a089c6a82eed203459fcf2e Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 10:56:14 -0500 Subject: [PATCH 10/48] Better to have location and @id match. We can consider whether the Location header and/or @id should be the /id/_id/expanded URI instead. However, downstream logic and behaviors expect the @id to come back for object validation. It is better to keep those aligned for now --- controllers/crud.js | 5 +++-- controllers/gog.js | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index c108de83..56e617c9 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -274,10 +274,11 @@ const idExpanded = async function (req, res, next) { // Annotations carrying multiple bodies are left out here const merged = annos.filter(anno => !Array.isArray(anno.body)) res.set('Annotations-Merged', String(merged.length)) - const expandedLocation = new URL(`/v1/id/${match._id}/expanded`, process.env.RERUM_ID_PREFIX).href let expanded = applyRawExpansion(match, annos) expanded = idNegotiation(expanded) - res.location(expandedLocation) + //const expandedLocation = new URL(`${match._id}/expanded`, process.env.RERUM_ID_PREFIX).href + //res.location(expandedLocation) + res.location(expanded["@id"] ?? expanded.id) res.json(expanded) } catch (error) { return next(utils.createExpressError(error)) diff --git a/controllers/gog.js b/controllers/gog.js index 2eb4ecdf..2f79dd6c 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -395,7 +395,6 @@ const expandedId = async function (req, res, next) { }) return next(utils.createExpressError(err)) } - const expandedLocation = new URL(`/gog/id/${match._id}`, process.env.RERUM_ID_PREFIX).href // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). res.set(utils.configureWebAnnoHeadersFor(match)) res.set("Cache-Control", "max-age=86400, must-revalidate") @@ -404,7 +403,9 @@ const expandedId = async function (req, res, next) { res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "") let expanded = await expand(match, generator) expanded = idNegotiation(expanded) - res.location(expandedLocation) + //const expandedLocation = new URL(`/gog/id/${match._id}`, process.env.RERUM_PREFIX).href + //res.location(expandedLocation) + res.location(expanded["@id"] ?? expanded.id) res.json(expanded) } catch (error) { return next(utils.createExpressError(error)) From 6d6c4d439df82aec5aba975fb7d927145c84f891 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 11:12:41 -0500 Subject: [PATCH 11/48] Paginate on the back end for the /expanded endpoints. Clients/browsers cannot be expected to paginate simple GET requests. The response is now fully assembled. --- __tests__/utils.test.js | 83 +++++++++++++++++++- controllers/crud.js | 9 ++- controllers/utils.js | 37 ++++++--- database/__mocks__/index.js | 1 + openapi/contracts/core-provider.openapi.yaml | 33 ++------ public/API.html | 21 ++--- 6 files changed, 128 insertions(+), 56 deletions(-) diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js index 900ca7c3..19fca238 100644 --- a/__tests__/utils.test.js +++ b/__tests__/utils.test.js @@ -9,7 +9,8 @@ import { parseDocumentID, _contextid, idNegotiation, - getPagination + getPagination, + findLeafAnnotationsFor } from '../controllers/utils.js' import { db, resetMocks } from '../database/index.js' @@ -288,3 +289,83 @@ describe('controllers/utils.js getPagination', () => { assert.ok(result.limit < huge, `limit should be clamped below ${huge}`) }) }) + +describe('controllers/utils.js findLeafAnnotationsFor', () => { + const targetURI = 'https://store.rerum.io/v1/id/entity' + + const buildAnnotations = (total) => Array.from({ length: total }, (_, i) => ({ + _id: `anno${String(i).padStart(5, '0')}`, + type: 'Annotation', + target: targetURI, + body: { position: i } + })) + + /** + * Stand in for the driver cursor so the gather can be watched round trip by round trip. + * Returns the {limit, skip} of each trip the gather actually made. + */ + const mockPagedFind = (stored) => { + const roundTrips = [] + db.find.mockImplementation(() => { + const state = { limit: stored.length, skip: 0 } + const cursor = { + sort: () => cursor, + limit: (value) => { state.limit = value; return cursor }, + skip: (value) => { state.skip = value; return cursor }, + toArray: async () => { + roundTrips.push({ limit: state.limit, skip: state.skip }) + return stored.slice(state.skip, state.skip + state.limit).map(anno => ({ ...anno })) + } + } + return cursor + }) + return roundTrips + } + + it('gathers every targeting Annotation, however many round trips that takes', async () => { + resetMocks() + const stored = buildAnnotations(1001) + const roundTrips = mockPagedFind(stored) + + const annos = await findLeafAnnotationsFor(targetURI) + + // The client asked once and is owed all 1001. There is no page to hand back. + assert.strictEqual(annos.length, stored.length) + assert.deepStrictEqual(annos.map(anno => anno.body.position), stored.map((_, i) => i)) + const batchSize = roundTrips[0].limit + assert.ok(batchSize < stored.length, 'this case must exceed one batch to exercise the walk') + assert.ok(roundTrips.length > 1, 'gathering more than a batch takes more than one round trip') + // Each trip skips past everything gathered so far, so nothing is read twice or missed. + const expectedSkips = roundTrips.map((_, i) => Math.min(i * batchSize, stored.length)) + assert.deepStrictEqual(roundTrips.map(trip => trip.skip), expectedSkips) + }) + + it('strips the internal _id from every gathered Annotation', async () => { + resetMocks() + mockPagedFind(buildAnnotations(3)) + + const annos = await findLeafAnnotationsFor(targetURI) + + assert.strictEqual(annos.length, 3) + assert.ok(annos.every(anno => !Object.hasOwn(anno, '_id')), '_id is internal and never returned') + }) + + it('makes a single round trip when the first batch is short', async () => { + resetMocks() + const roundTrips = mockPagedFind(buildAnnotations(3)) + + await findLeafAnnotationsFor(targetURI) + + assert.strictEqual(roundTrips.length, 1) + }) + + it('returns an empty Array when nothing targets the entity', async () => { + resetMocks() + const roundTrips = mockPagedFind([]) + + const annos = await findLeafAnnotationsFor(targetURI) + + assert.deepStrictEqual(annos, []) + assert.strictEqual(roundTrips.length, 1) + }) +}) diff --git a/controllers/crud.js b/controllers/crud.js index 56e617c9..e5212ac6 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -227,13 +227,13 @@ function applyRawExpansion(primitiveEntity, annos) { * * GET recognizes the '?generator=' and '?creator=' convenience parameters only. * POST reads literal MongoDB filter keys from the JSON body and ignores URL parameters as filters. - * Both methods page the Annotation search with the usual '?limit=' and '?skip=' parameters. + * Neither method pages. A client asks once and receives the entity assembled from every current + * Annotation targeting it -- findLeafAnnotationsFor() does whatever gathering that takes. * */ const idExpanded = async function (req, res, next) { res.set("Content-Type", "application/json; charset=utf-8") const id = req.params["_id"] const isPost = req.method === "POST" - const pagination = getPagination(req.query, 200) let filters = {} if (isPost) { //Express leaves the body undefined when a POST supplies none. That is an unfiltered expand. @@ -267,8 +267,9 @@ const idExpanded = async function (req, res, next) { // Include current version for optimistic locking res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "") const targetId = match["@id"] ?? match.id - const annos = targetId ? await findLeafAnnotationsFor(targetId, filters, pagination) : [] - // Let clients detect a full page. When this equals the limit there may be more to gather. + const annos = targetId ? await findLeafAnnotationsFor(targetId, filters) : [] + // Informational only. Every current Annotation targeting the entity is gathered, so this + // is the whole count and never a partial one. res.set('Annotations-Gathered', String(annos.length)) // How many of the gathered Annotations could contribute, which is a different number. // Annotations carrying multiple bodies are left out here diff --git a/controllers/utils.js b/controllers/utils.js index d3c7e12a..67db1272 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -12,6 +12,13 @@ const ObjectID = newID const MAX_QUERY_LIMIT = Number.parseInt(process.env.RERUM_MAX_QUERY_LIMIT ?? 500, 10) const MAX_QUERY_SKIP = Number.parseInt(process.env.RERUM_MAX_QUERY_SKIP ?? 100000, 10) +/** + * How many Annotations findLeafAnnotationsFor() pulls per round trip while gathering. + * This is an internal batch size, not a client facing page size. The gather does not stop until + * the database has no more matches, so this only decides how many round trips that takes. + */ +const EXPANSION_BATCH_SIZE = 500 + function clampNonNegativeInt(value, fallback, max) { const parsed = Number.parseInt(value, 10) if (!Number.isFinite(parsed) || parsed <= 0) return fallback @@ -145,12 +152,15 @@ function escapeRegex(literal) { * - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"} * - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"} * + * Every match is gathered. The caller expands with all of them, so this walks the result set in + * batches of EXPANSION_BATCH_SIZE until the database has no more to give. There is no page for a + * client to ask for and no truncation to report -- an expansion of 1000 Annotations gathers 1000. + * * @param targetId The '@id' or 'id' URI of the entity being expanded. * @param filters Literal MongoDB filter keys to AND into the query. - * @param pagination A {limit, skip} pair from getPagination(). - * @return An Array of matching Annotation documents, with '_id' removed. + * @return An Array of every matching Annotation document, with '_id' removed. */ -const findLeafAnnotationsFor = async function (targetId, filters = {}, pagination = null) { +const findLeafAnnotationsFor = async function (targetId, filters = {}) { // '$and' is always present so the filter conditions below can push into it from either branch. const queryObj = { "__rerum.history.next": { $exists: true, $size: 0 }, @@ -190,14 +200,19 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}, paginatio } queryObj["$and"].push({ [key]: value }) } - // Get the Annotations targeting this Entity from the db. Remove _id property. - let cursor = db.find(queryObj) - if (pagination) cursor = cursor.sort({ "_id": 1 }).limit(pagination.limit).skip(pagination.skip) - const matches = await cursor.toArray() - return matches.map(o => { - delete o._id - return o - }) + // Get every Annotation targeting this Entity from the db. Remove _id property. + // Sorting on '_id' keeps the walk stable across the round trips, since a batch is skipped past + // by how many Annotations have already been gathered. + const matches = [] + let batch = [] + do { + batch = await db.find(queryObj).sort({ "_id": 1 }).limit(EXPANSION_BATCH_SIZE).skip(matches.length).toArray() + for (const anno of batch) { + delete anno._id + matches.push(anno) + } + } while (batch.length === EXPANSION_BATCH_SIZE) + return matches } // Handle index actions diff --git a/database/__mocks__/index.js b/database/__mocks__/index.js index 51ec6a2c..b155179a 100644 --- a/database/__mocks__/index.js +++ b/database/__mocks__/index.js @@ -41,6 +41,7 @@ function createMockFunction(implementation = () => undefined) { function createCursor() { return { + sort: createMockFunction(function () { return this }), limit: createMockFunction(function () { return this }), skip: createMockFunction(function () { return this }), toArray: createMockFunction(() => Promise.resolve([])) diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index fa603009..1807f701 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -47,6 +47,10 @@ paths: /id/{id}/expanded: get: summary: Read object by id with its current Annotations merged in + description: >- + Not paged. Every current Annotation targeting the entity is gathered server side before the + expanded entity is assembled, so a client makes a single request. The 'Annotations-Gathered' + and 'Annotations-Merged' response headers report complete counts. operationId: getExpandedObjectById parameters: - $ref: '#/components/parameters/ObjectId' @@ -62,18 +66,6 @@ paths: description: Only expand with Annotations attributed to this creator. schema: type: string - - in: query - name: limit - required: false - description: Maximum Annotations to gather. Defaults to 200. - schema: - type: integer - - in: query - name: skip - required: false - description: Annotations to skip before gathering. Defaults to 0. - schema: - type: integer responses: '200': description: Expanded object payload @@ -98,24 +90,13 @@ paths: operationId: postExpandedObjectById description: >- The request body is an object of literal MongoDB filter properties ANDed into the search for - Annotations targeting the entity. URL query parameters supply no filters, though 'limit' and - 'skip' still page the search. The leaf version, the Annotation type, and the target + Annotations targeting the entity. URL query parameters supply no filters. Like the GET, the + search is not paged -- every Annotation matching the filters is gathered server side before + the expanded entity is assembled. The leaf version, the Annotation type, and the target constraints are applied automatically and cannot be overruled, so 'target', 'type', '@type', and '__rerum.history' keys are ignored. parameters: - $ref: '#/components/parameters/ObjectId' - - in: query - name: limit - required: false - description: Maximum Annotations to gather. Defaults to 200. - schema: - type: integer - - in: query - name: skip - required: false - description: Annotations to skip before gathering. Defaults to 0. - schema: - type: integer requestBody: required: false content: diff --git a/public/API.html b/public/API.html index 54f3bec3..0691be91 100644 --- a/public/API.html +++ b/public/API.html @@ -182,27 +182,21 @@

    Expanded record by id

    https spellings of the URI are matched.
  • ?creator—optional. Only expand with Annotations attributed to this creator.
    • -
  • ?limit and - ?skip—optional. Page the - Annotation search. limit defaults to 200.
  • Response: {JSON}—The record with identifier _id, plus the merged properties.
  • Response header Annotations-Gathered—how many - Annotations this expansion gathered, which is the size of the page you received. When it equals your - limit there may be more, and the record you - received was expanded from only part of its Annotations. Raise the - limit or walk pages with - skip.
    • + Annotations this expansion gathered. There is no paging on this endpoint. RERUM gathers + every current Annotation targeting the record before it assembles the response, however many that is, so + this is a complete count and never a partial one. It is informational—a client makes one request and + receives the fully expanded record.
  • Response header Annotations-Merged—how many of those gathered Annotations could contribute to the record. It is always less than or equal to Annotations-Gathered, and Annotations with multiple bodies are not counted, as that form is not read yet. A counted Annotation may still assert - nothing mergeable, so this is not a count of the properties you received either. Use - Annotations-Gathered, not this one, to decide - whether to keep paging.
    • + nothing mergeable, so this is not a count of the properties you received either.

    Only the current (leaf) versions of Annotations are gathered. No token is required. @@ -889,9 +883,8 @@

    Expanded record with filters

  • URL parameters supply no filters. The ?generator and ?creator parameters of the GET have no effect - here—supply them in the body instead. Paging is not a filter, so - ?limit and - ?skip still work, and the + here—supply them in the body instead. Like the GET, this is not paged: every Annotation matching your + filters is gathered before the response is assembled, and the Annotations-Gathered and Annotations-Merged response headers are set exactly as they are on the GET.
    • From 6ef2db314fed2cb010779a65b47325df17cc5451 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 11:17:07 -0500 Subject: [PATCH 12/48] No tests yet --- __tests__/utils.test.js | 83 +---------------------------------------- 1 file changed, 1 insertion(+), 82 deletions(-) diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js index 19fca238..900ca7c3 100644 --- a/__tests__/utils.test.js +++ b/__tests__/utils.test.js @@ -9,8 +9,7 @@ import { parseDocumentID, _contextid, idNegotiation, - getPagination, - findLeafAnnotationsFor + getPagination } from '../controllers/utils.js' import { db, resetMocks } from '../database/index.js' @@ -289,83 +288,3 @@ describe('controllers/utils.js getPagination', () => { assert.ok(result.limit < huge, `limit should be clamped below ${huge}`) }) }) - -describe('controllers/utils.js findLeafAnnotationsFor', () => { - const targetURI = 'https://store.rerum.io/v1/id/entity' - - const buildAnnotations = (total) => Array.from({ length: total }, (_, i) => ({ - _id: `anno${String(i).padStart(5, '0')}`, - type: 'Annotation', - target: targetURI, - body: { position: i } - })) - - /** - * Stand in for the driver cursor so the gather can be watched round trip by round trip. - * Returns the {limit, skip} of each trip the gather actually made. - */ - const mockPagedFind = (stored) => { - const roundTrips = [] - db.find.mockImplementation(() => { - const state = { limit: stored.length, skip: 0 } - const cursor = { - sort: () => cursor, - limit: (value) => { state.limit = value; return cursor }, - skip: (value) => { state.skip = value; return cursor }, - toArray: async () => { - roundTrips.push({ limit: state.limit, skip: state.skip }) - return stored.slice(state.skip, state.skip + state.limit).map(anno => ({ ...anno })) - } - } - return cursor - }) - return roundTrips - } - - it('gathers every targeting Annotation, however many round trips that takes', async () => { - resetMocks() - const stored = buildAnnotations(1001) - const roundTrips = mockPagedFind(stored) - - const annos = await findLeafAnnotationsFor(targetURI) - - // The client asked once and is owed all 1001. There is no page to hand back. - assert.strictEqual(annos.length, stored.length) - assert.deepStrictEqual(annos.map(anno => anno.body.position), stored.map((_, i) => i)) - const batchSize = roundTrips[0].limit - assert.ok(batchSize < stored.length, 'this case must exceed one batch to exercise the walk') - assert.ok(roundTrips.length > 1, 'gathering more than a batch takes more than one round trip') - // Each trip skips past everything gathered so far, so nothing is read twice or missed. - const expectedSkips = roundTrips.map((_, i) => Math.min(i * batchSize, stored.length)) - assert.deepStrictEqual(roundTrips.map(trip => trip.skip), expectedSkips) - }) - - it('strips the internal _id from every gathered Annotation', async () => { - resetMocks() - mockPagedFind(buildAnnotations(3)) - - const annos = await findLeafAnnotationsFor(targetURI) - - assert.strictEqual(annos.length, 3) - assert.ok(annos.every(anno => !Object.hasOwn(anno, '_id')), '_id is internal and never returned') - }) - - it('makes a single round trip when the first batch is short', async () => { - resetMocks() - const roundTrips = mockPagedFind(buildAnnotations(3)) - - await findLeafAnnotationsFor(targetURI) - - assert.strictEqual(roundTrips.length, 1) - }) - - it('returns an empty Array when nothing targets the entity', async () => { - resetMocks() - const roundTrips = mockPagedFind([]) - - const annos = await findLeafAnnotationsFor(targetURI) - - assert.deepStrictEqual(annos, []) - assert.strictEqual(roundTrips.length, 1) - }) -}) From 3c49b78c726585cb59a3b3b33a899efedd621fe8 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 11:18:12 -0500 Subject: [PATCH 13/48] No tests yet --- database/__mocks__/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/database/__mocks__/index.js b/database/__mocks__/index.js index b155179a..51ec6a2c 100644 --- a/database/__mocks__/index.js +++ b/database/__mocks__/index.js @@ -41,7 +41,6 @@ function createMockFunction(implementation = () => undefined) { function createCursor() { return { - sort: createMockFunction(function () { return this }), limit: createMockFunction(function () { return this }), skip: createMockFunction(function () { return this }), toArray: createMockFunction(() => Promise.resolve([])) From 145018391910db2d30ed56996b1e5aade1625b84 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 12:18:24 -0500 Subject: [PATCH 14/48] Manual cleanup --- controllers/crud.js | 44 +++++++++++++++++--------------------------- controllers/gog.js | 6 ++++-- controllers/utils.js | 8 +------- 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index e5212ac6..b6160129 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -127,25 +127,18 @@ const id = async function (req, res, next) { } } -/** - * The expand job always constrains the Annotations it gathers to the leaf versions, to the - * Annotation types, and to the entity in the request URI. A client cannot influence those, so - * these keys are dropped from a supplied filter body by exact name or dotted prefix. - */ -const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"] - -/** - * Identity and system properties an Annotation body must never overwrite. '@id' and '@context' - * are read by idNegotiation() and res.location() - */ -const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"]) - /** * Reduce a supplied POST body to the literal MongoDB filter keys the expand job will honor. * @param supplied The parsed JSON request body. * @return An object of filter keys, minus the ones this endpoint owns. */ function sanitizeExpansionFilters(supplied) { + /** + * The expand job always constrains the Annotations it gathers to the leaf versions, to the + * Annotation types, and to the entity in the request URI. A client cannot influence those, so + * these keys are dropped from a supplied filter body by exact name or dotted prefix. + */ + const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"] const filters = {} for (const [key, value] of Object.entries(supplied)) { if (RESERVED_FILTER_KEYS.some(reserved => key === reserved || key.startsWith(`${reserved}.`))) continue @@ -154,11 +147,6 @@ function sanitizeExpansionFilters(supplied) { return filters } -/** - * The Annotation body types whose value is kept whole rather than read as a single assertion. - */ -const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www.w3.org/ns/oa#TextualBody"]) - /** * The [key, value] assertions an Annotation makes about the entity it targets. * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other @@ -175,6 +163,10 @@ const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www * @return An Array of [key, value] pairs to merge onto the entity. */ function assertionsFrom(anno) { + /** + * The Annotation body types whose value is kept whole rather than read as a single assertion. + */ + const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www.w3.org/ns/oa#TextualBody"]) const assertions = [] if (typeof anno.bodyValue === "string") assertions.push(["bodyValue", anno.bodyValue]) const body = anno.body @@ -202,6 +194,8 @@ function assertionsFrom(anno) { * @return A new, expanded entity object. */ function applyRawExpansion(primitiveEntity, annos) { + //Identity and system properties an Annotation body must never overwrite. '@id' and '@context' + const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"]) const expandedEntity = structuredClone(primitiveEntity) // Hold __rerum aside so it can be re-appended after the merged properties. It will be the last property. const rerumProp = expandedEntity.__rerum @@ -248,7 +242,7 @@ const idExpanded = async function (req, res, next) { filters = sanitizeExpansionFilters(supplied) } else { - //Repeated query parameters arrive as an Array, which is not a filter value we support. + // Repeated query parameters arrive as an Array, which is not a filter value we will apply. if (typeof req.query.generator === "string" && req.query.generator) filters["__rerum.generatedBy"] = req.query.generator if (typeof req.query.creator === "string" && req.query.creator) filters.creator = req.query.creator } @@ -268,17 +262,13 @@ const idExpanded = async function (req, res, next) { res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "") const targetId = match["@id"] ?? match.id const annos = targetId ? await findLeafAnnotationsFor(targetId, filters) : [] - // Informational only. Every current Annotation targeting the entity is gathered, so this - // is the whole count and never a partial one. + // Every leaf Annotation matching the filter is gathered. This is the count. res.set('Annotations-Gathered', String(annos.length)) - // How many of the gathered Annotations could contribute, which is a different number. - // Annotations carrying multiple bodies are left out here - const merged = annos.filter(anno => !Array.isArray(anno.body)) + // How many of the Annotations contribute an assertion. May be less than annotations gathered. + const merged = annos.filter(anno => assertionsFrom(anno).length > 0) res.set('Annotations-Merged', String(merged.length)) - let expanded = applyRawExpansion(match, annos) + let expanded = applyRawExpansion(match, merged) expanded = idNegotiation(expanded) - //const expandedLocation = new URL(`${match._id}/expanded`, process.env.RERUM_ID_PREFIX).href - //res.location(expandedLocation) res.location(expanded["@id"] ?? expanded.id) res.json(expanded) } catch (error) { diff --git a/controllers/gog.js b/controllers/gog.js index 2f79dd6c..0d3a3c9b 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -341,6 +341,9 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde // Mirror DEER's client-side expand() (deer-utils.js buildValueObject) // When more than one current Annotation asserts the same key, collect the values into an Array. let expandedEntity = structuredClone(primitiveEntity) + // Hold __rerum aside so it can be re-appended after the merged properties. It will be the last property. + const rerumProp = expandedEntity.__rerum + delete expandedEntity.__rerum for(const anno of matches){ const body = anno.body // Annotations carrying multiple bodies are not expanded with. @@ -367,6 +370,7 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde expandedEntity[key] = valueObject } } + if(rerumProp !== undefined) expandedEntity.__rerum = rerumProp return expandedEntity } @@ -403,8 +407,6 @@ const expandedId = async function (req, res, next) { res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "") let expanded = await expand(match, generator) expanded = idNegotiation(expanded) - //const expandedLocation = new URL(`/gog/id/${match._id}`, process.env.RERUM_PREFIX).href - //res.location(expandedLocation) res.location(expanded["@id"] ?? expanded.id) res.json(expanded) } catch (error) { diff --git a/controllers/utils.js b/controllers/utils.js index 67db1272..607a0602 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -12,13 +12,6 @@ const ObjectID = newID const MAX_QUERY_LIMIT = Number.parseInt(process.env.RERUM_MAX_QUERY_LIMIT ?? 500, 10) const MAX_QUERY_SKIP = Number.parseInt(process.env.RERUM_MAX_QUERY_SKIP ?? 100000, 10) -/** - * How many Annotations findLeafAnnotationsFor() pulls per round trip while gathering. - * This is an internal batch size, not a client facing page size. The gather does not stop until - * the database has no more matches, so this only decides how many round trips that takes. - */ -const EXPANSION_BATCH_SIZE = 500 - function clampNonNegativeInt(value, fallback, max) { const parsed = Number.parseInt(value, 10) if (!Number.isFinite(parsed) || parsed <= 0) return fallback @@ -161,6 +154,7 @@ function escapeRegex(literal) { * @return An Array of every matching Annotation document, with '_id' removed. */ const findLeafAnnotationsFor = async function (targetId, filters = {}) { + const EXPANSION_BATCH_SIZE = 200 // '$and' is always present so the filter conditions below can push into it from either branch. const queryObj = { "__rerum.history.next": { $exists: true, $size: 0 }, From 85f1b6b9c570ff61586fe1e776a8c2b2b44f0d86 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 13:02:50 -0500 Subject: [PATCH 15/48] Changes during review. Supporting Annotations whose body is an Array but only has one {} in it, as we can treat that the same way as a body that is just {} for expanding purposes. --- controllers/crud.js | 20 +++++++++++--------- controllers/gog.js | 7 +++++-- controllers/utils.js | 19 ++++++++++++++++--- public/API.html | 7 ++++++- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index b6160129..837c5ebc 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -6,7 +6,7 @@ */ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID, findLeafAnnotationsFor } from './utils.js' +import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' /** * Create a new Linked Open Data object in RERUM v1. @@ -158,6 +158,7 @@ function sanitizeExpansionFilters(supplied) { * - body: {'key': {...}} a single assertion, value kept as-is * - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive * - body: {'@type':'oa:TextualBody', ...} the OA prefixed spelling of the same + * - body: [{'key': 'value'}] one body, serialized as a JSON-LD Array * * @param anno An Annotation document. * @return An Array of [key, value] pairs to merge onto the entity. @@ -169,7 +170,9 @@ function assertionsFrom(anno) { const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www.w3.org/ns/oa#TextualBody"]) const assertions = [] if (typeof anno.bodyValue === "string") assertions.push(["bodyValue", anno.bodyValue]) - const body = anno.body + // In JSON-LD a one-element Array and the bare value are the same body, so unwrap it first. The + // check below is about how many bodies an Annotation carries, not how they were serialized. + const body = Array.isArray(anno.body) && anno.body.length === 1 ? anno.body[0] : anno.body // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an // external resource with no embedded value to expand with. if (Array.isArray(body) || !body || typeof body !== "object") return assertions @@ -190,18 +193,16 @@ function assertionsFrom(anno) { * When more than one current Annotation asserts the same key, or the entity already carries it, * the values collect into an Array. * @param primitiveEntity The unexpanded entity. - * @param annos The Annotations targeting it. + * @param annoAssertions An Array holding the [key, value] assertions read from each Annotation. * @return A new, expanded entity object. */ -function applyRawExpansion(primitiveEntity, annos) { - //Identity and system properties an Annotation body must never overwrite. '@id' and '@context' - const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"]) +function applyRawExpansion(primitiveEntity, annoAssertions) { const expandedEntity = structuredClone(primitiveEntity) // Hold __rerum aside so it can be re-appended after the merged properties. It will be the last property. const rerumProp = expandedEntity.__rerum delete expandedEntity.__rerum - for (const anno of annos) { - for (const [key, value] of assertionsFrom(anno)) { + for (const assertions of annoAssertions) { + for (const [key, value] of assertions) { if (PROTECTED_EXPANSION_KEYS.has(key)) continue if (!Object.hasOwn(expandedEntity, key)) { expandedEntity[key] = value @@ -264,8 +265,9 @@ const idExpanded = async function (req, res, next) { const annos = targetId ? await findLeafAnnotationsFor(targetId, filters) : [] // Every leaf Annotation matching the filter is gathered. This is the count. res.set('Annotations-Gathered', String(annos.length)) + // Read each Annotation once. These assertions are both the merged count and the merge itself. // How many of the Annotations contribute an assertion. May be less than annotations gathered. - const merged = annos.filter(anno => assertionsFrom(anno).length > 0) + const merged = annos.map(anno => assertionsFrom(anno)).filter(assertions => assertions.length > 0) res.set('Annotations-Merged', String(merged.length)) let expanded = applyRawExpansion(match, merged) expanded = idNegotiation(expanded) diff --git a/controllers/gog.js b/controllers/gog.js index 0d3a3c9b..ad2c550d 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -8,7 +8,7 @@ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor } from './utils.js' +import { ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' // The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different // agents; only the trailing id is compared, so either host spelling matches. @@ -345,12 +345,15 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde const rerumProp = expandedEntity.__rerum delete expandedEntity.__rerum for(const anno of matches){ - const body = anno.body + // In JSON-LD a one-element Array and the bare value are the same body, so unwrap it first. + const body = Array.isArray(anno.body) && anno.body.length === 1 ? anno.body[0] : anno.body // Annotations carrying multiple bodies are not expanded with. if(!body || typeof body !== "object" || Array.isArray(body)) continue const keys = Object.keys(body) if(keys.length !== 1) continue const key = keys[0] + // An Annotation body cannot overwrite the entity's identity or its system properties. + if(PROTECTED_EXPANSION_KEYS.has(key)) continue const assertion = body[key] const valueObject = { value: assertion?.value ?? assertion, diff --git a/controllers/utils.js b/controllers/utils.js index 607a0602..9f3a1f04 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -121,6 +121,13 @@ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) */ const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"] +/** + * Identity and system properties an Annotation body must never overwrite when its assertions are + * merged onto an entity. '__proto__' is not data -- emitting it would hand a prototype pollution + * vector to every client that parses the response. + */ +const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"]) + /** * Escape the RegExp metacharacters in a literal so it can be embedded in a pattern and match only * itself. A RERUM URI has at least the dots of its host to escape. @@ -195,12 +202,17 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}) { queryObj["$and"].push({ [key]: value }) } // Get every Annotation targeting this Entity from the db. Remove _id property. - // Sorting on '_id' keeps the walk stable across the round trips, since a batch is skipped past - // by how many Annotations have already been gathered. + // Sorting on '_id' keeps the walk stable across the round trips, and each batch resumes from + // the last '_id' seen rather than skipping past the ones already gathered. Skipping makes the + // db re-walk the whole prefix every round trip; resuming reads each Annotation exactly once. const matches = [] let batch = [] + let resumeAfter = null do { - batch = await db.find(queryObj).sort({ "_id": 1 }).limit(EXPANSION_BATCH_SIZE).skip(matches.length).toArray() + const batchQuery = resumeAfter === null ? queryObj : { ...queryObj, "_id": { $gt: resumeAfter } } + batch = await db.find(batchQuery).sort({ "_id": 1 }).limit(EXPANSION_BATCH_SIZE).toArray() + // Read the resume point before '_id' is dropped from the document. + if (batch.length > 0) resumeAfter = batch.at(-1)._id for (const anno of batch) { delete anno._id matches.push(anno) @@ -566,6 +578,7 @@ export { _contextid, idNegotiation, findLeafAnnotationsFor, + PROTECTED_EXPANSION_KEYS, getPagination, generateSlugId, index, diff --git a/public/API.html b/public/API.html index 0691be91..a9e7dc1e 100644 --- a/public/API.html +++ b/public/API.html @@ -212,7 +212,12 @@

    Expanded record by id

    that property. Values are passed through exactly as they appear—a body of {"text": {"value": "hello"}} puts {"value": "hello"} on the entity, not - "hello". + "hello". A + body serialized as a one-element Array is + the same single body in JSON-LD and is read the same way— + [{"text": {"value": "hello"}}] and + {"text": {"value": "hello"}} are + equivalent.
  • A bodyValue string, and a TextualBody body, both land under the property bodyValue. The From a62b63cc656032e43f1ca8c0ba36e4d910b3684c Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 13:10:00 -0500 Subject: [PATCH 16/48] comment cleanup --- controllers/crud.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index 837c5ebc..b91d762f 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -231,7 +231,7 @@ const idExpanded = async function (req, res, next) { const isPost = req.method === "POST" let filters = {} if (isPost) { - //Express leaves the body undefined when a POST supplies none. That is an unfiltered expand. + // Express leaves the body undefined when a POST supplies none. That is an unfiltered expand. const supplied = req.body ?? {} if (typeof supplied !== "object" || Array.isArray(supplied)) { const err = { @@ -257,7 +257,7 @@ const idExpanded = async function (req, res, next) { return next(utils.createExpressError(err)) } res.set(utils.configureWebAnnoHeadersFor(match)) - //Support built in browser caching. A POST response is not cacheable. + // Support built in browser caching. A POST response is not cacheable. if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate") // Include current version for optimistic locking res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "") From 5af111e559478188415b626976f4d3ffc106d9e5 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 13:36:30 -0500 Subject: [PATCH 17/48] changes during review --- controllers/crud.js | 9 +++++++-- controllers/gog.js | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index b91d762f..49f8c7aa 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -6,7 +6,7 @@ */ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' +import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' /** * Create a new Linked Open Data object in RERUM v1. @@ -158,6 +158,7 @@ function sanitizeExpansionFilters(supplied) { * - body: {'key': {...}} a single assertion, value kept as-is * - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive * - body: {'@type':'oa:TextualBody', ...} the OA prefixed spelling of the same + * - body: {'type':['TextualBody'], ...} the same body, type serialized as a JSON-LD Array * - body: [{'key': 'value'}] one body, serialized as a JSON-LD Array * * @param anno An Annotation document. @@ -176,7 +177,11 @@ function assertionsFrom(anno) { // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an // external resource with no embedded value to expand with. if (Array.isArray(body) || !body || typeof body !== "object") return assertions - if (TEXTUAL_BODY_TYPES.has(body.type ?? body["@type"])) { + // In JSON-LD a type is either a single value or an Array of them. The query in + // findLeafAnnotationsFor() matches both spellings, so both are read here too. + const bodyType = body.type ?? body["@type"] + const bodyTypes = Array.isArray(bodyType) ? bodyType : [bodyType] + if (bodyTypes.some(t => TEXTUAL_BODY_TYPES.has(t))) { assertions.push(["bodyValue", body]) return assertions } diff --git a/controllers/gog.js b/controllers/gog.js index ad2c550d..9a491145 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -8,7 +8,7 @@ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' +import { getAgentClaim, getPagination, idNegotiation, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' // The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different // agents; only the trailing id is compared, so either host spelling matches. From 4986623de21986dfcdad2a8953633983b15635b6 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 13:51:02 -0500 Subject: [PATCH 18/48] changes during review and comment cleanup --- controllers/crud.js | 9 ++++----- controllers/gog.js | 15 ++------------- controllers/utils.js | 13 ++++++------- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index 49f8c7aa..b85ecde5 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -201,7 +201,7 @@ function assertionsFrom(anno) { * @param annoAssertions An Array holding the [key, value] assertions read from each Annotation. * @return A new, expanded entity object. */ -function applyRawExpansion(primitiveEntity, annoAssertions) { +function applyExpansionAnnotations(primitiveEntity, annoAssertions) { const expandedEntity = structuredClone(primitiveEntity) // Hold __rerum aside so it can be re-appended after the merged properties. It will be the last property. const rerumProp = expandedEntity.__rerum @@ -227,8 +227,7 @@ function applyRawExpansion(primitiveEntity, annoAssertions) { * * GET recognizes the '?generator=' and '?creator=' convenience parameters only. * POST reads literal MongoDB filter keys from the JSON body and ignores URL parameters as filters. - * Neither method pages. A client asks once and receives the entity assembled from every current - * Annotation targeting it -- findLeafAnnotationsFor() does whatever gathering that takes. + * Neither method pages. A client asks once and receives the entity assembled. * */ const idExpanded = async function (req, res, next) { res.set("Content-Type", "application/json; charset=utf-8") @@ -274,9 +273,9 @@ const idExpanded = async function (req, res, next) { // How many of the Annotations contribute an assertion. May be less than annotations gathered. const merged = annos.map(anno => assertionsFrom(anno)).filter(assertions => assertions.length > 0) res.set('Annotations-Merged', String(merged.length)) - let expanded = applyRawExpansion(match, merged) + let expanded = applyExpansionAnnotations(match, merged) expanded = idNegotiation(expanded) - res.location(expanded["@id"] ?? expanded.id) + res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) res.json(expanded) } catch (error) { return next(utils.createExpressError(error)) diff --git a/controllers/gog.js b/controllers/gog.js index 9a491145..17408e4a 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -8,7 +8,7 @@ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { getAgentClaim, getPagination, idNegotiation, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' +import { _contextid, getAgentClaim, getPagination, idNegotiation, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' // The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different // agents; only the trailing id is compared, so either host spelling matches. @@ -312,13 +312,6 @@ const _gog_glosses_from_manuscript = async function (req, res, next) { * Find relevant Annotations targeting a primitive RERUM entity. This is a 'full' expand. * Add the descriptive information in the Annotation bodies to the primitive object. * -* Anticipate likely Annotation body formats -* - anno.body -* - anno.body.value -* -* Gathering the Annotations is findLeafAnnotationsFor() in ./utils.js -- see its docblock for the -* target and Annotation type forms that are recognized. This function only reads their bodies. -* * @param primitiveEntity - An existing RERUM object * @param GENERATOR - A registered RERUM app's User Agent * @param CREATOR - Some kind of string representing a specific user. Often combined with GENERATOR. @@ -329,16 +322,12 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde // An entity is expandable if it carries a URI under either '@id' or 'id'. if(!primitiveEntity?.["@id"] && !primitiveEntity?.id) return primitiveEntity const targetId = primitiveEntity["@id"] ?? primitiveEntity.id ?? "unknown" - // Only expand with data from a specific app and/or a specific creator. The shared helper - // applies the leaf, target, and Annotation type constraints and doubles these two URIs - // across the http/https spellings. const filters = {} if(GENERATOR) filters["__rerum.generatedBy"] = GENERATOR if(CREATOR) filters.creator = CREATOR const matches = await findLeafAnnotationsFor(targetId, filters) // Combine the Annotation bodies with the primitive object. - // Mirror DEER's client-side expand() (deer-utils.js buildValueObject) // When more than one current Annotation asserts the same key, collect the values into an Array. let expandedEntity = structuredClone(primitiveEntity) // Hold __rerum aside so it can be re-appended after the merged properties. It will be the last property. @@ -410,7 +399,7 @@ const expandedId = async function (req, res, next) { res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "") let expanded = await expand(match, generator) expanded = idNegotiation(expanded) - res.location(expanded["@id"] ?? expanded.id) + res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) res.json(expanded) } catch (error) { return next(utils.createExpressError(error)) diff --git a/controllers/utils.js b/controllers/utils.js index 9f3a1f04..56d1e991 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -152,9 +152,12 @@ function escapeRegex(literal) { * - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"} * - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"} * - * Every match is gathered. The caller expands with all of them, so this walks the result set in - * batches of EXPANSION_BATCH_SIZE until the database has no more to give. There is no page for a - * client to ask for and no truncation to report -- an expansion of 1000 Annotations gathers 1000. + * Every match is gathered. This walks the result set in batches of EXPANSION_BATCH_SIZE + * until the database has no more to give. An expansion of 1000 Annotations gathers 1000. + * + * Sorting on '_id' keeps the walk stable across the round trips, and each batch resumes from + * the last '_id' seen rather than skipping past the ones already gathered. Skipping makes the + * db re-walk the whole prefix every round trip; resuming reads each Annotation exactly once. * * @param targetId The '@id' or 'id' URI of the entity being expanded. * @param filters Literal MongoDB filter keys to AND into the query. @@ -201,10 +204,6 @@ const findLeafAnnotationsFor = async function (targetId, filters = {}) { } queryObj["$and"].push({ [key]: value }) } - // Get every Annotation targeting this Entity from the db. Remove _id property. - // Sorting on '_id' keeps the walk stable across the round trips, and each batch resumes from - // the last '_id' seen rather than skipping past the ones already gathered. Skipping makes the - // db re-walk the whole prefix every round trip; resuming reads each Annotation exactly once. const matches = [] let batch = [] let resumeAfter = null From 15e20d02563fa129ce974237f67bb91f4a4fce63 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Tue, 18 Aug 2026 14:43:19 -0500 Subject: [PATCH 19/48] Changes during review and API.html simplification --- controllers/crud.js | 10 ++-- controllers/gog.js | 6 +- public/API.html | 136 ++++++++------------------------------------ 3 files changed, 32 insertions(+), 120 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index b85ecde5..0cfc9025 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -177,8 +177,6 @@ function assertionsFrom(anno) { // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an // external resource with no embedded value to expand with. if (Array.isArray(body) || !body || typeof body !== "object") return assertions - // In JSON-LD a type is either a single value or an Array of them. The query in - // findLeafAnnotationsFor() matches both spellings, so both are read here too. const bodyType = body.type ?? body["@type"] const bodyTypes = Array.isArray(bodyType) ? bodyType : [bodyType] if (bodyTypes.some(t => TEXTUAL_BODY_TYPES.has(t))) { @@ -261,10 +259,6 @@ const idExpanded = async function (req, res, next) { return next(utils.createExpressError(err)) } res.set(utils.configureWebAnnoHeadersFor(match)) - // Support built in browser caching. A POST response is not cacheable. - if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate") - // Include current version for optimistic locking - res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "") const targetId = match["@id"] ?? match.id const annos = targetId ? await findLeafAnnotationsFor(targetId, filters) : [] // Every leaf Annotation matching the filter is gathered. This is the count. @@ -275,6 +269,10 @@ const idExpanded = async function (req, res, next) { res.set('Annotations-Merged', String(merged.length)) let expanded = applyExpansionAnnotations(match, merged) expanded = idNegotiation(expanded) + // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). + if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate") + // Include current version for optimistic locking + res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "") res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) res.json(expanded) } catch (error) { diff --git a/controllers/gog.js b/controllers/gog.js index 17408e4a..9c3cb466 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -391,14 +391,14 @@ const expandedId = async function (req, res, next) { }) return next(utils.createExpressError(err)) } - // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). res.set(utils.configureWebAnnoHeadersFor(match)) + let expanded = await expand(match, generator) + expanded = idNegotiation(expanded) + // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). res.set("Cache-Control", "max-age=86400, must-revalidate") // No Last-Modified here, unlike GET /v1/id/:_id. It would compare against the root entity // before expand() merges the targeting Annotations. res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "") - let expanded = await expand(match, generator) - expanded = idNegotiation(expanded) res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) res.json(expanded) } catch (error) { diff --git a/public/API.html b/public/API.html index a9e7dc1e..90918538 100644 --- a/public/API.html +++ b/public/API.html @@ -185,68 +185,12 @@

    Expanded record by id

  • Response: {JSON}—The record with identifier _id, plus the merged properties.
    • -
  • Response header - Annotations-Gathered—how many - Annotations this expansion gathered. There is no paging on this endpoint. RERUM gathers - every current Annotation targeting the record before it assembles the response, however many that is, so - this is a complete count and never a partial one. It is informational—a client makes one request and - receives the fully expanded record.
    • -
  • Response header - Annotations-Merged—how many of - those gathered Annotations could contribute to the record. It is always less than or equal to - Annotations-Gathered, and Annotations with - multiple bodies are not counted, as that form is not read yet. A counted Annotation may still assert - nothing mergeable, so this is not a count of the properties you received either.

    • Only the current (leaf) versions of Annotations are gathered. No token is required. The response is a raw assembly—if your application needs the data in a particular shape, format the response for your own internal needs.

    -
    What gets merged
    -
      -
    • Only the Annotation's body and - bodyValue are read. Every other property of - the Annotation is ignored, and an Annotation carrying neither is ignored entirely.
      • -
    • A body with exactly one property contributes - that property. Values are passed through exactly as they appear—a body of - {"text": {"value": "hello"}} puts - {"value": "hello"} on the entity, not - "hello". A - body serialized as a one-element Array is - the same single body in JSON-LD and is read the same way— - [{"text": {"value": "hello"}}] and - {"text": {"value": "hello"}} are - equivalent.
      • -
    • A bodyValue string, and a - TextualBody body, both land under the property - bodyValue. The - TextualBody is kept whole so its - format and - language survive.
      • -
    • When several Annotations assert the same property, or the record already carries it, the values collect - into an Array. Array order is not guaranteed.
      • -
    • A property merged from a single Annotation is that value as-is; the same property merged from two or - more is an Array. Treat every merged property as possibly-an-Array—the shape depends on how many - Annotations exist, which changes over the record's life.
      • -
    • Identity and system properties are never overwritten: - @id, - id, - _id, - __rerum, - __deleted, and - @context. An assertion naming - __proto__ is dropped for the same reason—it - is not data, and emitting it would hand a prototype pollution vector to every client that parses the - response.
      • -
    • Skipped for now, as future work: a body with - more than one property, an Annotation with multiple bodies, the - Choice, - Composite, and - List constructs, and a - body that is a URI referencing an external - resource.
      • -

    Javascript Example
    
@@ -885,62 +829,8 @@ 
    Expanded record with filters
    
                 the real ones stored on an Annotation, so filtering by generating application means supplying
                 __rerum.generatedBy. An empty or absent body
                 expands without any filter.
-            
  • URL parameters supply no filters. The
-                ?generator and
-                ?creator parameters of the GET have no effect
-                here—supply them in the body instead. Like the GET, this is not paged: every Annotation matching your
-                filters is gathered before the response is assembled, and the
-                Annotations-Gathered and
-                Annotations-Merged response headers are set
-                exactly as they are on the GET.
    • 
-            
  • __rerum.generatedBy and
-                creator are matched against both the
-                http and
-                https spellings of the URI you provide. Every
-                other property is applied exactly as supplied.
    • 
-            
  • The Content-Type header must be
-                application/json. A body that is not a JSON
-                object is a 400.
    • 
+            
  • 
         
-        
    
-            Three constraints belong to the endpoint and cannot be overruled. Supplying them is not an error—they are
-            ignored, by exact name or as a dotted prefix.
-        
    
-        
-            
-                
-                    
-                    
-                
-            
-            
-                
-                    
-                    
-                
-                
-                    
-                    
-                
-                
-                    
-                    
-                
-            
-        
    IgnoredAlways applied instead
    target,
-                        target.@id,
-                        target.id,
-                        target.source,
-                        target.source.@id,
-                        target.source.idAnnotations targeting the record at
-                        _id, including those
-                        targeting it through a
-                        SpecificResource or through a
-                        fragment of its URI such as
-                        #xywh=0,0,100,100
    type,
-                        @typeAnnotations only
    __rerum.history.next,
-                        __rerum.history.previous,
-                        __rerum.history.primeCurrent (leaf) versions only
    
         
    
             
    Javascript Example
    
             
    
@@ -1715,6 +1605,30 @@ 
    __rerum Property Explained
    
             existing vocabularies, but for now the applications accessing RERUM will
             need to interpret this data if it is relevant.
         
    
+        
    Entity Expansion
    
+        
    
+            Only the Annotation's body and bodyValue are read. Every other property of the Annotation is ignored, and an Annotation carrying neither is ignored entirely.
+        
    
+        
    
+            A body with exactly one property contributes that property. Values are passed through exactly as they appear—a body of
+            {"text": {"value": "hello"}} puts
+            {"value": "hello"} on the entity, not
+            "hello".
+            A body serialized as a one-element Array is the same single body in JSON-LD and is read the same way—
+            [{"text": {"value": "hello"}}] and
+            {"text": {"value": "hello"}} are equivalent.
+        
    
+        
    
+            A bodyValue string, and a
+            TextualBody body, both land under the property
+            bodyValue. The
+            TextualBody is kept whole so its
+            format and
+            language survive.
+        
    
+        
    
+            When several Annotations assert the same property, or the record already carries it, the values collect into an Array. Array order is not guaranteed.
+        
        
         
    History
    
         
    History is stored through pointers that create a B-Tree. All nodes in the B-tree know the root node, the
             previous node, and the next node(s).
    

From 0fa58093caea4ddb6c689b22708574c00d5fc82e Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Tue, 18 Aug 2026 15:11:10 -0500
Subject: [PATCH 20/48] Changes during review

---
 controllers/crud.js  | 10 +++++++--
 controllers/utils.js | 51 +++++++++++++++++++++++++++++---------------
 public/API.html      |  9 +++++++-
 3 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index 0cfc9025..ce7e7918 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -157,7 +157,7 @@ function sanitizeExpansionFilters(supplied) {
  *   - body: {'key': 'value'}                             a single assertion
  *   - body: {'key': {...}}                               a single assertion, value kept as-is
  *   - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive
- *   - body: {'@type':'oa:TextualBody', ...}              the OA prefixed spelling of the same
+ *   - body: {'@type':'oa:TextualBody', ...}              the 'oa:' prefixed spelling of the same W3C class
  *   - body: {'type':['TextualBody'], ...}                the same body, type serialized as a JSON-LD Array
  *   - body: [{'key': 'value'}]                           one body, serialized as a JSON-LD Array
  *
@@ -260,7 +260,13 @@ const idExpanded = async function (req, res, next) {
         }
         res.set(utils.configureWebAnnoHeadersFor(match))
         const targetId = match["@id"] ?? match.id
-        const annos = targetId ? await findLeafAnnotationsFor(targetId, filters) : []
+        // A record minted with a Slug also resolves at '/id/', so an Annotation may
+        // target it by that URI instead of by its '@id'.  The slug URI is built off the entity's own
+        // URI rather than RERUM_ID_PREFIX so a record minted under a legacy host keeps that host.
+        const slug = match.__rerum?.slug
+        const lastSlash = targetId?.lastIndexOf("/") ?? -1
+        const slugTargetId = slug && lastSlash !== -1 ? targetId.slice(0, lastSlash + 1) + slug : undefined
+        const annos = await findLeafAnnotationsFor([targetId, slugTargetId], filters)
         // Every leaf Annotation matching the filter is gathered.  This is the count.
         res.set('Annotations-Gathered', String(annos.length))
         // Read each Annotation once.  These assertions are both the merged count and the merge itself.
diff --git a/controllers/utils.js b/controllers/utils.js
index 56d1e991..a6338efa 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -149,8 +149,19 @@ function escapeRegex(literal) {
  *   - target: {'source':{'id':'uri'}}                           a SpecificResource with an embedded source
  *   - target: 'uri#xywh=0,0,100,100'                            a fragment of the resource
  * and the likely Annotation type formats
- *   - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"}
- *   - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
+ *   - {"type": "Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"}
+ *   - {"@type": "Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
+ *
+ * Only Web Annotation Data Model era Annotations are expanded with.  'oa:Annotation' is the Open
+ * Annotation era spelling carried by IIIF Presentation 2.1 data, which names its target under 'on'
+ * and its body under 'resource', so there is nothing here that could read one.  The full
+ * 'http://www.w3.org/ns/oa#Annotation' IRI is kept because that is the W3C class itself, what a
+ * fully expanded JSON-LD Web Annotation carries.
+ *
+ * An entity can answer to more than one URI.  A record minted with a Slug resolves at both
+ * '/id/<_id>' and '/id/', and an Annotation may legitimately target it by
+ * either one, so a caller hands over every URI the entity is known by.  An Annotation matching any
+ * of them targets this entity and is gathered.
  *
  * Every match is gathered.  This walks the result set in batches of EXPANSION_BATCH_SIZE 
  * until the database has no more to give.  An expansion of 1000 Annotations gathers 1000.
@@ -159,41 +170,47 @@ function escapeRegex(literal) {
  * the last '_id' seen rather than skipping past the ones already gathered.  Skipping makes the
  * db re-walk the whole prefix every round trip; resuming reads each Annotation exactly once.
  *
- * @param targetId The '@id' or 'id' URI of the entity being expanded.
+ * @param targetIds The '@id' or 'id' URI of the entity being expanded, or an Array of the URIs it
+ * is known by when it answers to more than one.
  * @param filters Literal MongoDB filter keys to AND into the query.
  * @return An Array of every matching Annotation document, with '_id' removed.
  */
-const findLeafAnnotationsFor = async function (targetId, filters = {}) {
+const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
     const EXPANSION_BATCH_SIZE = 200
-    // '$and' is always present so the filter conditions below can push into it from either branch.
+    const targetURIs = (Array.isArray(targetIds) ? targetIds : [targetIds]).filter(Boolean)
+    // Nothing to target is nothing to gather.  An empty '$or' is a MongoDB error, not an empty result.
+    if (targetURIs.length === 0) return []
+    // '$and' is always present so the target, type, and filter conditions can push into it.
     const queryObj = {
         "__rerum.history.next": { $exists: true, $size: 0 },
         "$and": []
     }
     const annoTypeConditions = [
-        {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"},
-        {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
+        {"type": "Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"},
+        {"@type": "Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
     ]
-    if (/^https?:\/\//.test(targetId)) {
-        const targetConditions = []
+    // Every URI the entity answers to contributes its conditions to the same '$or'.
+    const targetConditions = []
+    for (const targetURI of targetURIs) {
+        if (!/^https?:\/\//.test(targetURI)) {
+            // Not a URI, so there is no http/https spelling or fragment of it to anticipate.
+            targetConditions.push({ "target": targetURI })
+            continue
+        }
         // Hanging a fragment off the URI is the other W3C way to target part of a resource rather
         // than the whole of it, and an exact match will not catch one.
         const fragmentPatterns = ["http", "https"].map(scheme =>
-            new RegExp(`^${escapeRegex(targetId.replace(/^https?/, scheme))}#`)
+            new RegExp(`^${escapeRegex(targetURI.replace(/^https?/, scheme))}#`)
         )
         // 'target.source' is the W3C SpecificResource, which is how an Annotation targets a
         // selected region of a resource rather than the whole of it.
         for (const targetKey of TARGET_KEYS) {
-            targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") })
-            targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") })
+            targetConditions.push({ [targetKey]: targetURI.replace(/^https?/, "http") })
+            targetConditions.push({ [targetKey]: targetURI.replace(/^https?/, "https") })
             for (const fragmentPattern of fragmentPatterns) targetConditions.push({ [targetKey]: fragmentPattern })
         }
-        queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions})
-    }
-    else {
-        queryObj["$and"].push({"$or": annoTypeConditions})
-        queryObj.target = targetId
     }
+    queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions})
     for (const [key, value] of Object.entries(filters)) {
         if (URI_DOUBLED_FILTER_KEYS.has(key) && typeof value === "string" && /^https?:\/\//.test(value)) {
             queryObj["$and"].push({"$or": [
diff --git a/public/API.html b/public/API.html
index 90918538..22869ba2 100644
--- a/public/API.html
+++ b/public/API.html
@@ -187,7 +187,14 @@ 
    Expanded record by id
    
                 properties.
    • 
         
         
    
-            Only the current (leaf) versions of Annotations are gathered. No token is required.
+            Only Annotations of the
+            W3C Web Annotation Data Model are
+            gathered, and only their current (leaf) versions. The Open Annotation era
+            oa:Annotation carried by IIIF Presentation API
+            2.1⚠️
+            data names its target under on and its body under
+            resource, so it is not expanded with. No token is
+            required.
             The response is a raw assembly—if your application needs the data in a particular shape, format the
             response for your own internal needs.
         
    

From dfb0e1d813ed985c817a99de4af9872c5d0a692d Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Tue, 18 Aug 2026 16:53:11 -0500
Subject: [PATCH 21/48] Changes during review

---
 controllers/utils.js | 21 ++++++++++++---------
 public/API.html      | 20 ++++++++++++++------
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/controllers/utils.js b/controllers/utils.js
index a6338efa..577bb2ac 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -149,14 +149,17 @@ function escapeRegex(literal) {
  *   - target: {'source':{'id':'uri'}}                           a SpecificResource with an embedded source
  *   - target: 'uri#xywh=0,0,100,100'                            a fragment of the resource
  * and the likely Annotation type formats
- *   - {"type": "Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"}
- *   - {"@type": "Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
+ *   - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"}
+ *   - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
  *
- * Only Web Annotation Data Model era Annotations are expanded with.  'oa:Annotation' is the Open
- * Annotation era spelling carried by IIIF Presentation 2.1 data, which names its target under 'on'
- * and its body under 'resource', so there is nothing here that could read one.  The full
- * 'http://www.w3.org/ns/oa#Annotation' IRI is kept because that is the W3C class itself, what a
- * fully expanded JSON-LD Web Annotation carries.
+ * All three spellings are the same class.  The W3C Web Annotation context defines the 'oa' prefix as
+ * 'http://www.w3.org/ns/oa#' and the term 'Annotation' as 'oa:Annotation', so a compacted, a prefixed,
+ * and a fully expanded JSON-LD Web Annotation all name 'http://www.w3.org/ns/oa#Annotation'.
+ *
+ * Only 'target' and 'body' are read, whatever the type spelling says.  That is what keeps Open
+ * Annotation era data out on its own -- IIIF Presentation 2.1 carries 'oa:Annotation' but names its
+ * target under 'on' and its body under 'resource', neither of which is read here, so such a record
+ * matches nothing and expands with nothing.
  *
  * An entity can answer to more than one URI.  A record minted with a Slug resolves at both
  * '/id/<_id>' and '/id/', and an Annotation may legitimately target it by
@@ -186,8 +189,8 @@ const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
         "$and": []
     }
     const annoTypeConditions = [
-        {"type": "Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"},
-        {"@type": "Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
+        {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"},
+        {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
     ]
     // Every URI the entity answers to contributes its conditions to the same '$or'.
     const targetConditions = []
diff --git a/public/API.html b/public/API.html
index 22869ba2..00468919 100644
--- a/public/API.html
+++ b/public/API.html
@@ -189,11 +189,17 @@ 
    Expanded record by id
    
         
    
             Only Annotations of the
             W3C Web Annotation Data Model are
-            gathered, and only their current (leaf) versions. The Open Annotation era
-            oa:Annotation carried by IIIF Presentation API
+            gathered, and only their current (leaf) versions. An Annotation is recognized by
+            Annotation,
+            oa:Annotation, or
+            http://www.w3.org/ns/oa#Annotation under either
+            type or
+            @type—the W3C context defines all three as
+            the same class. Only target and
+            body are read, so IIIF Presentation API
             2.1⚠️
-            data names its target under on and its body under
-            resource, so it is not expanded with. No token is
+            data, which names its target under on and its body
+            under resource, is not expanded with. No token is
             required.
             The response is a raw assembly—if your application needs the data in a particular shape, format the
             response for your own internal needs.
@@ -836,7 +842,9 @@ 
    Expanded record with filters
    
                 the real ones stored on an Annotation, so filtering by generating application means supplying
                 __rerum.generatedBy. An empty or absent body
                 expands without any filter.
-            
  • 
+            
  • Response: {JSON}—The
+                record with identifier _id, plus the merged
+                properties from the Annotations matching the supplied filters.
    • 
         
         
    
             
    Javascript Example
    
@@ -1612,7 +1620,7 @@ 
    __rerum Property Explained
    
             existing vocabularies, but for now the applications accessing RERUM will
             need to interpret this data if it is relevant.
         
    
-        
    Entity Expansion
    
+        
    Entity Expansion
    
         
    
             Only the Annotation's body and bodyValue are read. Every other property of the Annotation is ignored, and an Annotation carrying neither is ignored entirely.
         
    

From a1cd2448dffbb4460ede38ab9a421891212e1d70 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Tue, 18 Aug 2026 17:20:09 -0500
Subject: [PATCH 22/48] Code documentation and API documentation simplification

---
 controllers/crud.js  |  1 -
 controllers/gog.js   |  4 +---
 controllers/utils.js | 22 ++++------------------
 public/API.html      | 24 ++++++------------------
 4 files changed, 11 insertions(+), 40 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index ce7e7918..9aa9c502 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -201,7 +201,6 @@ function assertionsFrom(anno) {
  */
 function applyExpansionAnnotations(primitiveEntity, annoAssertions) {
     const expandedEntity = structuredClone(primitiveEntity)
-    // Hold __rerum aside so it can be re-appended after the merged properties. It will be the last property.
     const rerumProp = expandedEntity.__rerum
     delete expandedEntity.__rerum
     for (const assertions of annoAssertions) {
diff --git a/controllers/gog.js b/controllers/gog.js
index 9c3cb466..e61decda 100644
--- a/controllers/gog.js
+++ b/controllers/gog.js
@@ -330,7 +330,6 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde
     // Combine the Annotation bodies with the primitive object.
     // When more than one current Annotation asserts the same key, collect the values into an Array.
     let expandedEntity = structuredClone(primitiveEntity)
-    // Hold __rerum aside so it can be re-appended after the merged properties. It will be the last property.
     const rerumProp = expandedEntity.__rerum
     delete expandedEntity.__rerum
     for(const anno of matches){
@@ -396,8 +395,7 @@ const expandedId = async function (req, res, next) {
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         res.set("Cache-Control", "max-age=86400, must-revalidate")
-        // No Last-Modified here, unlike GET /v1/id/:_id.  It would compare against the root entity
-        // before expand() merges the targeting Annotations.
+        // Include current version for optimistic locking
         res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "")
         res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"])
         res.json(expanded)
diff --git a/controllers/utils.js b/controllers/utils.js
index 577bb2ac..df5a648a 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -152,26 +152,12 @@ function escapeRegex(literal) {
  *   - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"}
  *   - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
  *
- * All three spellings are the same class.  The W3C Web Annotation context defines the 'oa' prefix as
- * 'http://www.w3.org/ns/oa#' and the term 'Annotation' as 'oa:Annotation', so a compacted, a prefixed,
- * and a fully expanded JSON-LD Web Annotation all name 'http://www.w3.org/ns/oa#Annotation'.
+ * Only 'target' and 'body' are read, whatever the type spelling says.  
  *
- * Only 'target' and 'body' are read, whatever the type spelling says.  That is what keeps Open
- * Annotation era data out on its own -- IIIF Presentation 2.1 carries 'oa:Annotation' but names its
- * target under 'on' and its body under 'resource', neither of which is read here, so such a record
- * matches nothing and expands with nothing.
- *
- * An entity can answer to more than one URI.  A record minted with a Slug resolves at both
+ * Any entity can answer to more than one URI.  A record minted with a Slug resolves at both
  * '/id/<_id>' and '/id/', and an Annotation may legitimately target it by
- * either one, so a caller hands over every URI the entity is known by.  An Annotation matching any
- * of them targets this entity and is gathered.
- *
- * Every match is gathered.  This walks the result set in batches of EXPANSION_BATCH_SIZE 
- * until the database has no more to give.  An expansion of 1000 Annotations gathers 1000.
- *
- * Sorting on '_id' keeps the walk stable across the round trips, and each batch resumes from
- * the last '_id' seen rather than skipping past the ones already gathered.  Skipping makes the
- * db re-walk the whole prefix every round trip; resuming reads each Annotation exactly once.
+ * either one.  Every match is gathered.  This walks the result set in batches of EXPANSION_BATCH_SIZE 
+ * until the database has no more to give.
  *
  * @param targetIds The '@id' or 'id' URI of the entity being expanded, or an Array of the URIs it
  * is known by when it answers to more than one.
diff --git a/public/API.html b/public/API.html
index 00468919..a2d984ee 100644
--- a/public/API.html
+++ b/public/API.html
@@ -169,9 +169,7 @@ 
    Expanded record by id
    
             
         
         
    
-            Gather every current Annotation targeting the record and merge what those Annotations assert onto it.
-            This does the entity assembly that client applications otherwise perform with many
-            /query requests.
+            Gather every current Annotation targeting the record and merge what those Annotations assert onto it.  This does the entity assembly that client applications otherwise perform with many  /query requests.
         
    
         
      
             
    • _id—the id of the record in
@@ -186,21 +184,10 @@ 
      Expanded record by id
      
                 with identifier _id, plus the merged
                 properties.
      • 
         
    
+        
    
+            Annotation objects must use target and body properties to be gathered.  See Entity Expansion for more information.
+        
    
         
    
-            Only Annotations of the
-            W3C Web Annotation Data Model are
-            gathered, and only their current (leaf) versions. An Annotation is recognized by
-            Annotation,
-            oa:Annotation, or
-            http://www.w3.org/ns/oa#Annotation under either
-            type or
-            @type—the W3C context defines all three as
-            the same class. Only target and
-            body are read, so IIIF Presentation API
-            2.1⚠️
-            data, which names its target under on and its body
-            under resource, is not expanded with. No token is
-            required.
             The response is a raw assembly—if your application needs the data in a particular shape, format the
             response for your own internal needs.
         
    
@@ -1621,8 +1608,9 @@ 
    __rerum Property Explained
    
             need to interpret this data if it is relevant.
         
    
         
    Entity Expansion
    
+
         
    
-            Only the Annotation's body and bodyValue are read. Every other property of the Annotation is ignored, and an Annotation carrying neither is ignored entirely.
+            Only Annotations of the W3C Web Annotation Data Model are gathered, and only their latest (leaf) versions from history.  Only the Annotation's body and bodyValue are read. Every other property of the Annotation is ignored, and an Annotation carrying neither is ignored entirely.
         
    
         
    
             A body with exactly one property contributes that property. Values are passed through exactly as they appear—a body of

From 82c7e274a486710c5f7e696f4b1d9e686537d5c5 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 10:29:29 -0500
Subject: [PATCH 23/48] API file fixes

---
 public/API.html | 60 ++++++++++++++++++++++++-------------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/public/API.html b/public/API.html
index a2d984ee..2a9dd008 100644
--- a/public/API.html
+++ b/public/API.html
@@ -41,7 +41,7 @@ 
    
         
         
    API (1.1.0)
    
         
      
-            
    • API (1.1.0)
+            
    • API (1.1.0)
                 
         
      
-            Annotation objects must use target and body properties to be gathered.  See Entity Expansion for more information.
+            Annotation objects must use a target property to be gathered, and a body or bodyValue property to contribute.  See Entity Expansion for more information.
         
      
         
      
             The response is a raw assembly—if your application needs the data in a particular shape, format the
@@ -362,9 +362,9 @@ 
      Create
      
         
      
             Add a completely new object to RERUM and receive the Location URI
             as a response header and the complete RERUM record as the response body. Accepts only single JSON objects in the request body.
-            
      
-                The __rerum, @id and _id properties are ignored.  In cases where the Linked Data @context property maps '@id' to 'id', the id property is also ignored.
      
         
      
+        
      
+            The __rerum, @id and _id properties are ignored.  In cases where the Linked Data @context property maps '@id' to 'id', the id property is also ignored.
         
      
         
      
             
      Javascript Example
      
@@ -372,7 +372,7 @@ 
      Create
      
                 const saved_obj = await fetch("https://devstore.rerum.io/v1/api/create", {
                     method: "POST",
                     headers:{
-                        "Authorization": "Bearer eyJz93a...k4laUWw" 
+                        "Authorization": "Bearer eyJz93a...k4laUWw",
                          "Content-Type": "application/json; charset=utf-8"
                     },
                     body: JSON.stringify({
@@ -380,7 +380,7 @@ 
      Create
      
                     })
                 })
                 .then(resp => resp.json())
-                .catch(err => {throw err})
+                .catch(err => {throw err})

    @@ -407,8 +407,7 @@

    Bulk Create

    /bulkCreate [{JSON}] - 201 - [{JSON}] + 201 [{JSON}] @@ -432,16 +431,16 @@

    Bulk Create

    const saved_objs = await fetch("https://devstore.rerum.io/v1/api/bulkCreate", { method: "POST", headers:{ - "Authorization": "Bearer eyJz93a...k4laUWw" + "Authorization": "Bearer eyJz93a...k4laUWw", "Content-Type": "application/json; charset=utf-8" }, - body: JSON.stringify([ + body: JSON.stringify([ {"hello": "sun"}, {"goodbye": "moon"} ]) }) .then(resp => resp.json()) - .catch(err => {throw err}) + .catch(err => {throw err})

    @@ -998,8 +997,7 @@

    Bulk Update

    /bulkUpdate [{JSON}] - 200 - [{JSON}] + 200 [{JSON}] @@ -1022,9 +1020,10 @@

    Bulk Update

    const updated_objs = await fetch("https://devstore.rerum.io/v1/api/bulkUpdate", { method: "PUT", headers:{ - "Authorization": "Bearer eyJz93a...k4laUWw" + "Authorization": "Bearer eyJz93a...k4laUWw", "Content-Type": "application/json; charset=utf-8" - body: JSON.stringify([{ + }, + body: JSON.stringify([{ "@id": "https://devstore.rerum.io/v1/id/abcdef1234567890", "hello": "new world" }, @@ -1032,8 +1031,9 @@

    Bulk Update

    "@id": "https://devstore.rerum.io/v1/id/1234567890abcdef", "goodbye": "old planet" }]) + }) .then(resp => resp.json()) - .catch(err => {throw err}) + .catch(err => {throw err})

    @@ -1055,7 +1055,7 @@

    Bulk Update

    Overwrite

    - RERUM allows the Generator of a record to overwrite that record. An error will be returned if the agent encoded in the request "Authorization" access token does not match the agent of the existing record. + RERUM allows the Generator of a record to overwrite that record. An error will be returned if the agent encoded in the request "Authorization" access token does not match the __rerum.generatedBy agent of the existing record. Replace a record using a reference to its internal RERUM id and receive the Location URI for the resulting record as a response header and the complete record as the response body. This will have the effects of update, set, and unset actions. New keys will be created and keys not present in the request will not be present in the resulting record. The record is replaced in place, or overwritten, and so the @id or id will not change and the history connected with this record will not be altered. The __rerum.isOverwritten property will be set to the date and time of the overwrite. @@ -1220,10 +1220,10 @@

    Patch Update

    }

    - Note that resp.headers.get("location") will return "https://devstore.rerum.io/v1/id/1234567890abcdef" + Note that resp.headers.get("location") will return "https://devstore.rerum.io/v1/id/1234567890abcdef"

    - Note that the original record "https://devstore.rerum.io/v1/id/abcdef1234567890" now has + Note that the original record "https://devstore.rerum.io/v1/id/abcdef1234567890" now has __rerum.history.next : ["https://devstore.rerum.io/v1/id/1234567890abcdef"] @@ -1299,10 +1299,10 @@

    Add Properties

    }

    - Note that resp.headers.get("location") will return "https://devstore.rerum.io/v1/id/1234567890abcdef" + Note that resp.headers.get("location") will return "https://devstore.rerum.io/v1/id/1234567890abcdef"

    - Note that the original record "https://devstore.rerum.io/v1/id/abcdef1234567890" now has + Note that the original record "https://devstore.rerum.io/v1/id/abcdef1234567890" now has __rerum.history.next : ["https://devstore.rerum.io/v1/id/1234567890abcdef"] @@ -1378,10 +1378,10 @@

    Remove Properties

    }

    - Note that resp.headers.get("location") will return "https://devstore.rerum.io/v1/id/1234567890abcdef" + Note that resp.headers.get("location") will return "https://devstore.rerum.io/v1/id/1234567890abcdef"

    - Note that the original record "https://devstore.rerum.io/v1/id/abcdef1234567890" now has + Note that the original record "https://devstore.rerum.io/v1/id/abcdef1234567890" now has __rerum.history.next : ["https://devstore.rerum.io/v1/id/1234567890abcdef"] @@ -1444,13 +1444,13 @@

    RERUM released

    }

    - Note that resp.headers.get("location") will return "https://devstore.rerum.io/v1/id/abcdef1234567890" + Note that resp.headers.get("location") will return "https://devstore.rerum.io/v1/id/abcdef1234567890"

    DELETE

    - RERUM allows the Generator of a record to delete that record. An error will be returned if the agent encoded in the request "Authorization" access token does not match the agent of the existing record. - RERUM DELETE does not remove anything from the server. Deleted records are only marked as deleted. + RERUM allows the Generator of a record to delete that record. An error will be returned if the agent encoded in the request "Authorization" access token does not match the __rerum.generatedBy agent of the existing record. + RERUM DELETE does not remove anything from the server. Deleted records are only marked as deleted. Records marked as deleted do not return in query results and may only be directly retrieved by @id or id.

    Deleted records are removed from history trees. RERUM will do this automatically when a record is deleted. This cannot be undone. @@ -1631,11 +1631,11 @@

    Entity Expansion

    When several Annotations assert the same property, or the record already carries it, the values collect into an Array. Array order is not guaranteed. -

    +

    History

    History is stored through pointers that create a B-Tree. All nodes in the B-tree know the root node, the previous node, and the next node(s).

    -

    You can ask for all descendants or all ancestors from any given node so long as you know the node’s @id or id and the node has not been deleted. See history parents and history +

    You can ask for all descendants or all ancestors from any given node so long as you know the node’s @id or id and the node has not been deleted. See history parents and history children for more details about this process.

    Deleted records are not present in any B-Tree, but do exist as separate nodes that can be requested by the URI directly. A snapshot of their position at the time of deletion persists in these deleted nodes.

    @@ -1678,12 +1678,12 @@

    Web Annotation

    RERUM Responses

    The intention of the API is to follow RESTful practices. These practices drive what requests we accept and what responses we have to the various scenarios around those requests. What it means to be RESTful varies - wildly, but our efforts follow the guidelines at https://www.restapitutorial.com/resources.html

    + wildly, but our efforts follow the guidelines at REST API Tutorial.

    RERUM follows REST, IIIF and Web Annotation standards to form its responses to users. For more information about why RERUM chose a certain HTTP status code see the graph below.

    alt text

    If you are confused as to what type of requests give what response, review the Web - Annotation and RESTful standards.

    + Annotation and RESTful standards.

    This site is open source. Improve this page. From 4deb2170f98473d79aa201048dfaeb47cecff259 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Wed, 19 Aug 2026 10:47:52 -0500 Subject: [PATCH 24/48] changes and cleanup during review --- controllers/crud.js | 2 +- controllers/utils.js | 11 +++---- openapi/contracts/core-provider.openapi.yaml | 30 ++++++++++++-------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index 9aa9c502..fdcb4fc3 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -193,7 +193,7 @@ function assertionsFrom(anno) { /** * Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values. - * When more than one current Annotation asserts the same key, or the entity already carries it, + * When more than one current Annotation asserts the same key, or the entity already carries it, * the values collect into an Array. * @param primitiveEntity The unexpanded entity. * @param annoAssertions An Array holding the [key, value] assertions read from each Annotation. diff --git a/controllers/utils.js b/controllers/utils.js index df5a648a..1bd72914 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -112,13 +112,13 @@ const generateSlugId = async function(slug_id="", next){ /** * RERUM has minted these two under both 'http' and 'https' over the years, so a filter on either * must match both spellings. Every other supplied filter key is applied exactly as given. - */ + */ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) /** * The properties an Annotation can carry the URI of its target under. * Choice, Composite, and List target constructs, whose members sit in an 'items' Array. They are not supported here. - */ + */ const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"] /** @@ -152,11 +152,11 @@ function escapeRegex(literal) { * - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"} * - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"} * - * Only 'target' and 'body' are read, whatever the type spelling says. + * Only 'target' and 'body' are read, whatever the type spelling says. * * Any entity can answer to more than one URI. A record minted with a Slug resolves at both * '/id/<_id>' and '/id/', and an Annotation may legitimately target it by - * either one. Every match is gathered. This walks the result set in batches of EXPANSION_BATCH_SIZE + * either one. Every match is gathered. This walks the result set in batches of EXPANSION_BATCH_SIZE * until the database has no more to give. * * @param targetIds The '@id' or 'id' URI of the entity being expanded, or an Array of the URIs it @@ -166,7 +166,8 @@ function escapeRegex(literal) { */ const findLeafAnnotationsFor = async function (targetIds, filters = {}) { const EXPANSION_BATCH_SIZE = 200 - const targetURIs = (Array.isArray(targetIds) ? targetIds : [targetIds]).filter(Boolean) + // Deduped -- a record whose slug equals its _id would otherwise push every condition twice. + const targetURIs = [...new Set((Array.isArray(targetIds) ? targetIds : [targetIds]).filter(Boolean))] // Nothing to target is nothing to gather. An empty '$or' is a MongoDB error, not an empty result. if (targetURIs.length === 0) return [] // '$and' is always present so the target, type, and filter conditions can push into it. diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index 1807f701..48443094 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -54,18 +54,8 @@ paths: operationId: getExpandedObjectById parameters: - $ref: '#/components/parameters/ObjectId' - - in: query - name: generator - required: false - description: Only expand with Annotations generated by this registered app agent. - schema: - type: string - - in: query - name: creator - required: false - description: Only expand with Annotations attributed to this creator. - schema: - type: string + - $ref: '#/components/parameters/ExpansionGenerator' + - $ref: '#/components/parameters/ExpansionCreator' responses: '200': description: Expanded object payload @@ -80,6 +70,8 @@ paths: operationId: headExpandedObjectById parameters: - $ref: '#/components/parameters/ObjectId' + - $ref: '#/components/parameters/ExpansionGenerator' + - $ref: '#/components/parameters/ExpansionCreator' responses: '200': description: Expanded object headers @@ -611,6 +603,20 @@ components: required: true schema: type: string + ExpansionGenerator: + in: query + name: generator + required: false + description: Only expand with Annotations generated by this registered app agent. + schema: + type: string + ExpansionCreator: + in: query + name: creator + required: false + description: Only expand with Annotations attributed to this creator. + schema: + type: string schemas: GenericObject: type: object From c1d1297838312ed315238e6e07e47585b2041453 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Wed, 19 Aug 2026 11:57:27 -0500 Subject: [PATCH 25/48] _id constraint, the resume-point merge, the dead clone removal, and _contextid guard related to latent bug. Documentation cleanup. --- controllers/crud.js | 3 +- controllers/gog.js | 2 +- controllers/utils.js | 28 +++-- openapi/contracts/core-provider.openapi.yaml | 55 +++++++--- public/API.html | 107 ++++++++++++++++++- 5 files changed, 167 insertions(+), 28 deletions(-) diff --git a/controllers/crud.js b/controllers/crud.js index fdcb4fc3..f629e7b9 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -260,8 +260,7 @@ const idExpanded = async function (req, res, next) { res.set(utils.configureWebAnnoHeadersFor(match)) const targetId = match["@id"] ?? match.id // A record minted with a Slug also resolves at '/id/', so an Annotation may - // target it by that URI instead of by its '@id'. The slug URI is built off the entity's own - // URI rather than RERUM_ID_PREFIX so a record minted under a legacy host keeps that host. + // target it by that URI instead of by its '@id'. const slug = match.__rerum?.slug const lastSlash = targetId?.lastIndexOf("/") ?? -1 const slugTargetId = slug && lastSlash !== -1 ? targetId.slice(0, lastSlash + 1) + slug : undefined diff --git a/controllers/gog.js b/controllers/gog.js index e61decda..2289ecb9 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -6,7 +6,7 @@ * @author cubap, thehabes */ -import { newID, isValidID, db } from '../database/index.js' +import { db } from '../database/index.js' import utils from '../utils.js' import { _contextid, getAgentClaim, getPagination, idNegotiation, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' diff --git a/controllers/utils.js b/controllers/utils.js index 1bd72914..e36ca069 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -30,7 +30,9 @@ function getPagination(query = {}, defaultLimit = 100) { /** * Check if a @context value contains a known @id-id mapping context * - * @param contextInput An Array of string URIs or a string URI. + * @param contextInput A string URI, or an Array of them. JSON-LD allows an '@context' Array to mix + * URI strings with inline term definition objects, so a member that is not a string is skipped -- + * only a string can name one of the known contexts. * @return A boolean */ function _contextid(contextInput) { @@ -46,6 +48,8 @@ function _contextid(contextInput) { ] if(Array.isArray(contextInput)) { for(const c of contextInput) { + // An inline term definition object, or any other non-string member, names no context. + if(typeof c !== "string") continue contextURI = c bool = knownContexts.some(contextCheck) if(bool) break @@ -70,14 +74,13 @@ const idNegotiation = function (resBody) { const _id = resBody._id delete resBody._id if(!resBody["@context"]) return resBody - let modifiedResBody = structuredClone(resBody) + // Only this path hands back the body it was given, so it is the only one that needs a copy of it. + if(!_contextid(resBody["@context"])) return structuredClone(resBody) + // The '@id' to 'id' path builds its result from scratch instead, and never reads a copy. const context = { "@context": resBody["@context"] } - if(_contextid(resBody["@context"])) { - delete resBody["@id"] - delete resBody["@context"] - modifiedResBody = Object.assign(context, { "id": process.env.RERUM_ID_PREFIX + _id }, resBody) - } - return modifiedResBody + delete resBody["@id"] + delete resBody["@context"] + return Object.assign(context, { "id": process.env.RERUM_ID_PREFIX + _id }, resBody) } /** @@ -166,13 +169,14 @@ function escapeRegex(literal) { */ const findLeafAnnotationsFor = async function (targetIds, filters = {}) { const EXPANSION_BATCH_SIZE = 200 - // Deduped -- a record whose slug equals its _id would otherwise push every condition twice. const targetURIs = [...new Set((Array.isArray(targetIds) ? targetIds : [targetIds]).filter(Boolean))] // Nothing to target is nothing to gather. An empty '$or' is a MongoDB error, not an empty result. if (targetURIs.length === 0) return [] // '$and' is always present so the target, type, and filter conditions can push into it. + // RERUM mints every '_id' as a hex string. The batch resume below pages on '_id'. const queryObj = { "__rerum.history.next": { $exists: true, $size: 0 }, + "_id": { $type: "string" }, "$and": [] } const annoTypeConditions = [ @@ -215,7 +219,11 @@ const findLeafAnnotationsFor = async function (targetIds, filters = {}) { let batch = [] let resumeAfter = null do { - const batchQuery = resumeAfter === null ? queryObj : { ...queryObj, "_id": { $gt: resumeAfter } } + // The resume point joins the '_id' conditions rather than replacing them, so the '$type' + // constraint survives into every batch after the first. + const batchQuery = resumeAfter === null + ? queryObj + : { ...queryObj, "_id": { ...queryObj["_id"], $gt: resumeAfter } } batch = await db.find(batchQuery).sort({ "_id": 1 }).limit(EXPANSION_BATCH_SIZE).toArray() // Read the resume point before '_id' is dropped from the document. if (batch.length > 0) resumeAfter = batch.at(-1)._id diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index 48443094..adddc16a 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -47,11 +47,6 @@ paths: /id/{id}/expanded: get: summary: Read object by id with its current Annotations merged in - description: >- - Not paged. Every current Annotation targeting the entity is gathered server side before the - expanded entity is assembled, so a client makes a single request. The 'Annotations-Gathered' - and 'Annotations-Merged' response headers report complete counts. - operationId: getExpandedObjectById parameters: - $ref: '#/components/parameters/ObjectId' - $ref: '#/components/parameters/ExpansionGenerator' @@ -59,6 +54,13 @@ paths: responses: '200': description: Expanded object payload + headers: + Annotations-Gathered: + $ref: '#/components/headers/AnnotationsGathered' + Annotations-Merged: + $ref: '#/components/headers/AnnotationsMerged' + Cache-Control: + $ref: '#/components/headers/ExpansionCacheControl' content: application/json: schema: @@ -75,18 +77,18 @@ paths: responses: '200': description: Expanded object headers + headers: + Annotations-Gathered: + $ref: '#/components/headers/AnnotationsGathered' + Annotations-Merged: + $ref: '#/components/headers/AnnotationsMerged' + Cache-Control: + $ref: '#/components/headers/ExpansionCacheControl' '404': $ref: '#/components/responses/NotFound' post: summary: Read object by id with its current Annotations merged in, filtered by the request body operationId: postExpandedObjectById - description: >- - The request body is an object of literal MongoDB filter properties ANDed into the search for - Annotations targeting the entity. URL query parameters supply no filters. Like the GET, the - search is not paged -- every Annotation matching the filters is gathered server side before - the expanded entity is assembled. The leaf version, the Annotation type, and the target - constraints are applied automatically and cannot be overruled, so 'target', 'type', '@type', - and '__rerum.history' keys are ignored. parameters: - $ref: '#/components/parameters/ObjectId' requestBody: @@ -98,6 +100,11 @@ paths: responses: '200': description: Expanded object payload + headers: + Annotations-Gathered: + $ref: '#/components/headers/AnnotationsGathered' + Annotations-Merged: + $ref: '#/components/headers/AnnotationsMerged' content: application/json: schema: @@ -596,6 +603,30 @@ paths: '409': $ref: '#/components/responses/Conflict' components: + headers: + AnnotationsGathered: + description: >- + How many current Annotations targeting the entity were gathered for this expansion. The + search is not paged, so this is a complete count rather than the size of a page. + schema: + type: integer + minimum: 0 + AnnotationsMerged: + description: >- + How many of the gathered Annotations could contribute an assertion. An Annotation carrying + multiple bodies, or a body with more than one property, is gathered but not counted here. A + counted Annotation may still assert only a protected property, so this is not a count of the + properties present on the expanded entity either. + schema: + type: integer + minimum: 0 + ExpansionCacheControl: + description: >- + Browser caching policy for the expanded entity. An entity's Annotations are volatile while it + is first created and stable afterward, so the expansion is cacheable for 24 hours. + schema: + type: string + example: max-age=86400, must-revalidate parameters: ObjectId: in: path diff --git a/public/API.html b/public/API.html index 2a9dd008..8b01dddc 100644 --- a/public/API.html +++ b/public/API.html @@ -84,6 +84,7 @@

    API (1.1.0)

  • DELETE
  • __rerum @@ -187,6 +188,21 @@

    Expanded record by id

    Annotation objects must use a target property to be gathered, and a body or bodyValue property to contribute. See Entity Expansion for more information.

    +

    + Supply ?generator and + ?creator at most once each. A parameter supplied + more than once is not applied, and the response is the same unfiltered expansion. +

    +

    + The response reports how much of the record's Annotation set went into it through the + Annotations-Gathered + and Annotations-Merged headers. +

    +

    + This response is browser cacheable for 24 hours—Cache-Control: max-age=86400, must-revalidate. + A record's Annotations are volatile while it is first being created and stable afterward, so this trades a + hard reload in the uncommon stale case for not re-running the expansion on every visit. +

    The response is a raw assembly—if your application needs the data in a particular shape, format the response for your own internal needs. @@ -828,10 +844,58 @@

    Expanded record with filters

    the real ones stored on an Annotation, so filtering by generating application means supplying __rerum.generatedBy. An empty or absent body expands without any filter.
    • -
  • Response: {JSON}—The +
  • Response: {JSON}—The record with identifier _id, plus the merged properties from the Annotations matching the supplied filters.
    • +

    + Every supplied filter is combined with AND, so a + filter can only narrow the set of Annotations. There is no way to widen it past the record named in the + request URI. +

    +

    + The expansion always constrains itself to the leaf Annotation versions, to the Annotation types, and to the + record in the request URI. A client cannot influence those, so the following keys are dropped from the + request body rather than applied. Sending one is not an error, but it will not filter anything. +

    + + + + + + + + + + + + + + + + + + + + + + + + + +
    Ignored keyWhy
    targetThe record in the request URI is the target.
    typeOnly Web Annotations are gathered.
    @typeOnly Web Annotations are gathered.
    __rerum.historyOnly the latest (leaf) version of an Annotation is gathered.
    +

    + Each is dropped by exact name or by dotted prefix, so + target.source and + __rerum.history.next are dropped along with + target and + __rerum.history. +

    +

    + This response carries the same + Annotations-Gathered + and Annotations-Merged headers as the GET. +

    Javascript Example
    
@@ -1617,7 +1681,7 @@ 
    Entity Expansion
    
             {"text": {"value": "hello"}} puts
             {"value": "hello"} on the entity, not
             "hello".
-            A body serialized as a one-element Array is the same single body in JSON-LD and is read the same way—
+            A body serialized as a one-element Array is the same single body in JSON-LD and is read the same way—
             [{"text": {"value": "hello"}}] and
             {"text": {"value": "hello"}} are equivalent.
         
    
@@ -1630,8 +1694,45 @@ 
    Entity Expansion
    
             language survive.
         
    
         
    
-            When several Annotations assert the same property, or the record already carries it, the values collect into an Array. Array order is not guaranteed.
+            When several Annotations assert the same property, or the record already carries it, the values collect into
+            an Array. Two Annotations asserting
+            ["red", "blue"] and
+            ["black", "white"] produce
+            ["red", "blue", "black", "white"]. 
+            Array order is not guaranteed.
+        
    
+        
    
+            This holds for a property the record already carries as an Array too, so an Annotation asserting something
+            like a IIIF items or
+            behavior appends into the record's own Array rather
+            than sitting beside it. The expansion is a raw assembly and does not preserve the boundary between what the
+            record said and what each Annotation added.
+        
    
+        
    
+            A record's identity and system properties are never overwritten by an Annotation. An Annotation body
+            asserting @id,
+            id,
+            _id,
+            @context,
+            __rerum,
+            __deleted, or
+            __proto__ contributes nothing. The expanded record
+            answers to the same URI as the record you asked for.
         
    
+        
    Expansion response headers
    
+        
    
+            Neither form of the request is paged. Every Annotation matching the filters is gathered before the record is
+            assembled, so both counts below are complete rather than the size of a page.
+        
    
+        
      
+            
    • Annotations-Gathered—how
+                many current Annotations targeting the record were gathered for this expansion.
      • 
+            
    • Annotations-Merged—how
+                many of those could contribute an assertion. An Annotation may carry an unprocessable 
+                body.  Though it is gathered
+                it is not counted here. A counted Annotation may still assert a protected property, so this is not a
+                count of the properties you received either.
      • 
+        
    
         
    History
    
         
    History is stored through pointers that create a B-Tree. All nodes in the B-tree know the root node, the
             previous node, and the next node(s).
    

From bf6c6851d7a7676bd344895192826ed8d1fd200a Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 13:00:19 -0500
Subject: [PATCH 26/48] Contract alignment and API.html touch ups

---
 openapi/contracts/core-provider.openapi.yaml | 65 ++++++++++++++++++++
 public/API.html                              | 18 +++---
 2 files changed, 75 insertions(+), 8 deletions(-)

diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml
index adddc16a..174bb534 100644
--- a/openapi/contracts/core-provider.openapi.yaml
+++ b/openapi/contracts/core-provider.openapi.yaml
@@ -28,6 +28,15 @@ paths:
       responses:
         '200':
           description: Object payload
+          headers:
+            Cache-Control:
+              $ref: '#/components/headers/RecordCacheControl'
+            Last-Modified:
+              $ref: '#/components/headers/RecordLastModified'
+            Location:
+              $ref: '#/components/headers/CanonicalLocation'
+            Current-Overwritten-Version:
+              $ref: '#/components/headers/CurrentOverwrittenVersion'
           content:
             application/json:
               schema:
@@ -42,11 +51,21 @@ paths:
       responses:
         '200':
           description: Object headers
+          headers:
+            Cache-Control:
+              $ref: '#/components/headers/RecordCacheControl'
+            Last-Modified:
+              $ref: '#/components/headers/RecordLastModified'
+            Location:
+              $ref: '#/components/headers/CanonicalLocation'
+            Current-Overwritten-Version:
+              $ref: '#/components/headers/CurrentOverwrittenVersion'
         '404':
           $ref: '#/components/responses/NotFound'
   /id/{id}/expanded:
     get:
       summary: Read object by id with its current Annotations merged in
+      operationId: getExpandedObjectById
       parameters:
         - $ref: '#/components/parameters/ObjectId'
         - $ref: '#/components/parameters/ExpansionGenerator'
@@ -61,12 +80,18 @@ paths:
               $ref: '#/components/headers/AnnotationsMerged'
             Cache-Control:
               $ref: '#/components/headers/ExpansionCacheControl'
+            Location:
+              $ref: '#/components/headers/CanonicalLocation'
+            Current-Overwritten-Version:
+              $ref: '#/components/headers/CurrentOverwrittenVersion'
           content:
             application/json:
               schema:
                 $ref: '#/components/schemas/GenericObject'
         '404':
           $ref: '#/components/responses/NotFound'
+        '405':
+          description: Method not allowed — this endpoint only permits GET, HEAD, and POST.
     head:
       summary: Read expanded object headers by id
       operationId: headExpandedObjectById
@@ -84,8 +109,14 @@ paths:
               $ref: '#/components/headers/AnnotationsMerged'
             Cache-Control:
               $ref: '#/components/headers/ExpansionCacheControl'
+            Location:
+              $ref: '#/components/headers/CanonicalLocation'
+            Current-Overwritten-Version:
+              $ref: '#/components/headers/CurrentOverwrittenVersion'
         '404':
           $ref: '#/components/responses/NotFound'
+        '405':
+          description: Method not allowed — this endpoint only permits GET, HEAD, and POST.
     post:
       summary: Read object by id with its current Annotations merged in, filtered by the request body
       operationId: postExpandedObjectById
@@ -105,6 +136,10 @@ paths:
               $ref: '#/components/headers/AnnotationsGathered'
             Annotations-Merged:
               $ref: '#/components/headers/AnnotationsMerged'
+            Location:
+              $ref: '#/components/headers/CanonicalLocation'
+            Current-Overwritten-Version:
+              $ref: '#/components/headers/CurrentOverwrittenVersion'
           content:
             application/json:
               schema:
@@ -113,6 +148,8 @@ paths:
           $ref: '#/components/responses/BadRequest'
         '404':
           $ref: '#/components/responses/NotFound'
+        '405':
+          description: Method not allowed — this endpoint only permits GET, HEAD, and POST.
         '413':
           $ref: '#/components/responses/PayloadTooLarge'
         '415':
@@ -627,6 +664,34 @@ components:
       schema:
         type: string
         example: max-age=86400, must-revalidate
+    RecordCacheControl:
+      description: >-
+        Browser caching policy for the record. A record is only replaced by an update, which mints a
+        new URI, so the response at this URI is cacheable for 24 hours.
+      schema:
+        type: string
+        example: max-age=86400, must-revalidate
+    RecordLastModified:
+      description: >-
+        The record's '__rerum.isOverwritten' time, or its '__rerum.createdAt' time when it has never
+        been overwritten, so a client can revalidate with 'If-Modified-Since'. Not sent by the
+        '/expanded' forms, where it would describe the record ahead of the Annotations merged into it.
+      schema:
+        type: string
+        format: date-time
+    CanonicalLocation:
+      description: >-
+        The canonical URI of the record, in its '_id' form. A request made by Slug is answered with
+        the stable '_id' URI here rather than the Slug it was asked for.
+      schema:
+        type: string
+    CurrentOverwrittenVersion:
+      description: >-
+        The record's '__rerum.isOverwritten' value, for optimistic locking. Empty when the record has
+        never been overwritten. On the '/expanded' forms this describes the record itself, not the
+        Annotations merged into it.
+      schema:
+        type: string
   parameters:
     ObjectId:
       in: path
diff --git a/public/API.html b/public/API.html
index 8b01dddc..8b6892f0 100644
--- a/public/API.html
+++ b/public/API.html
@@ -899,14 +899,16 @@ 
    Expanded record with filters
    
         
    
             
    Javascript Example
    
             
    
-                const expanded = await fetch("https://devstore.rerum.io/v1/id/11111/expanded", {
-                    method: "POST",
-                    headers: { "Content-Type": "application/json" },
-                    body: JSON.stringify({
-                        "__rerum.generatedBy": "https://devstore.rerum.io/v1/id/agent7",
-                        "motivation": "describing"
-                    })
-                }).then(resp => resp.json()).catch(err => {throw err})
+                const expanded = await fetch("https://devstore.rerum.io/v1/id/11111/expanded", {
+                    method: "POST",
+                    headers: { "Content-Type": "application/json" },
+                    body: JSON.stringify({
+                        "__rerum.generatedBy": "https://devstore.rerum.io/v1/id/agent7",
+                        "motivation": "describing"
+                    })
+                })
+                .then(resp => resp.json())
+                .catch(err => {throw err})
             
    
         
    
         
    HTTP POST Method Override
    

From 2925fe45d9f3b997774667858dd48e1fd6a7e0b9 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 13:04:45 -0500
Subject: [PATCH 27/48] Contract alignment and API.html touch ups

---
 openapi/contracts/core-provider.openapi.yaml | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml
index 174bb534..c08ad029 100644
--- a/openapi/contracts/core-provider.openapi.yaml
+++ b/openapi/contracts/core-provider.openapi.yaml
@@ -643,17 +643,13 @@ components:
   headers:
     AnnotationsGathered:
       description: >-
-        How many current Annotations targeting the entity were gathered for this expansion. The
-        search is not paged, so this is a complete count rather than the size of a page.
+        How many current Annotations targeting the entity were gathered for this expansion.
       schema:
         type: integer
         minimum: 0
     AnnotationsMerged:
       description: >-
-        How many of the gathered Annotations could contribute an assertion. An Annotation carrying
-        multiple bodies, or a body with more than one property, is gathered but not counted here. A
-        counted Annotation may still assert only a protected property, so this is not a count of the
-        properties present on the expanded entity either.
+        How many of the gathered Annotations could contribute an assertion.
       schema:
         type: integer
         minimum: 0
@@ -674,8 +670,7 @@ components:
     RecordLastModified:
       description: >-
         The record's '__rerum.isOverwritten' time, or its '__rerum.createdAt' time when it has never
-        been overwritten, so a client can revalidate with 'If-Modified-Since'. Not sent by the
-        '/expanded' forms, where it would describe the record ahead of the Annotations merged into it.
+        been overwritten, so a client can revalidate with 'If-Modified-Since'.
       schema:
         type: string
         format: date-time
@@ -688,8 +683,7 @@ components:
     CurrentOverwrittenVersion:
       description: >-
         The record's '__rerum.isOverwritten' value, for optimistic locking. Empty when the record has
-        never been overwritten. On the '/expanded' forms this describes the record itself, not the
-        Annotations merged into it.
+        never been overwritten.
       schema:
         type: string
   parameters:

From d6127016235857b16f6ccd25c7378b4dafeef50b Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 15:17:16 -0500
Subject: [PATCH 28/48] Deleted records are not expanded, and a client supplied
 __rerum is ignored

/expanded skips the Annotation gather and the merge for a deleted record and
hands the tombstone back exactly as GET /v1/id/:_id does, with both count
headers at 0.  A production record with 8 leaf Annotations targeting it was
being re-dressed with the descriptive data of the record it used to be.

configureRerumOptions only receives a trustworthy __rerum on the update path,
where the caller hands it a record from the database, so a supplied __rerum is
read there alone.  history and releases are built locally instead of being
rebound to the received objects, so caller properties under them no longer
reach storage and an affected record is cleaned on its next version.
---
 __tests__/utils.test.js                      | 54 ++++++++++++++++++++
 controllers/crud.js                          |  9 ++--
 controllers/utils.js                         |  1 -
 openapi/contracts/core-provider.openapi.yaml |  9 ++--
 public/API.html                              |  9 ++++
 utils.js                                     | 17 +++---
 6 files changed, 86 insertions(+), 13 deletions(-)

diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js
index 900ca7c3..3d862952 100644
--- a/__tests__/utils.test.js
+++ b/__tests__/utils.test.js
@@ -49,6 +49,60 @@ describe('utils.js configureRerumOptions', () => {
     )
     assert.strictEqual(result.__rerum.generatedBy, 'https://store.rerum.io/v1/id/legitimate-agent')
   })
+
+  it('drops caller-supplied properties inside __rerum.history and __rerum.releases', () => {
+    const result = utils.configureRerumOptions(
+      'agent-uri',
+      {
+        a: 'b',
+        __rerum: {
+          history: { t: 'h', prime: 'forged', next: ['forged'] },
+          releases: { evil: true, previous: 'p', replaces: 'forged' }
+        }
+      },
+      false,
+      false
+    )
+    assert.deepStrictEqual(Object.keys(result.__rerum.history).sort(), ['next', 'previous', 'prime'])
+    assert.deepStrictEqual(Object.keys(result.__rerum.releases).sort(), ['next', 'previous', 'replaces'])
+    assert.deepStrictEqual(result.__rerum.history, { next: [], previous: '', prime: 'root' })
+    // A create reads nothing out of a supplied __rerum, so the release pointer is not seeded either.
+    assert.deepStrictEqual(result.__rerum.releases, { next: [], previous: '', replaces: '' })
+  })
+
+  it('does not mutate the __rerum it was handed', () => {
+    const received = { __rerum: { history: { t: 'h' }, releases: { evil: true } } }
+    utils.configureRerumOptions('agent-uri', received, false, false)
+    assert.deepStrictEqual(received.__rerum.history, { t: 'h' })
+    assert.deepStrictEqual(received.__rerum.releases, { evil: true })
+  })
+
+  it('still carries prime and releases.previous forward on an update', () => {
+    const fromPrime = utils.configureRerumOptions(
+      'agent-uri',
+      { '@id': 'https://store.rerum.io/v1/id/AAA', __rerum: { history: { prime: 'root', previous: '', next: [] } } },
+      true,
+      false
+    )
+    assert.strictEqual(fromPrime.__rerum.history.prime, 'https://store.rerum.io/v1/id/AAA')
+    assert.strictEqual(fromPrime.__rerum.history.previous, 'https://store.rerum.io/v1/id/AAA')
+
+    const fromChild = utils.configureRerumOptions(
+      'agent-uri',
+      {
+        '@id': 'https://store.rerum.io/v1/id/BBB',
+        __rerum: {
+          history: { prime: 'https://store.rerum.io/v1/id/AAA', previous: '', next: [] },
+          releases: { previous: 'https://store.rerum.io/v1/id/REL', next: [], replaces: '' }
+        }
+      },
+      true,
+      false
+    )
+    assert.strictEqual(fromChild.__rerum.history.prime, 'https://store.rerum.io/v1/id/AAA')
+    assert.strictEqual(fromChild.__rerum.history.previous, 'https://store.rerum.io/v1/id/BBB')
+    assert.strictEqual(fromChild.__rerum.releases.previous, 'https://store.rerum.io/v1/id/REL')
+  })
 })
 
 describe('controllers/utils.js generateSlugId', () => {
diff --git a/controllers/crud.js b/controllers/crud.js
index f629e7b9..2d878178 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -258,20 +258,23 @@ const idExpanded = async function (req, res, next) {
             return next(utils.createExpressError(err))
         }
         res.set(utils.configureWebAnnoHeadersFor(match))
+        // A deleted record is a tombstone -- whatever it used to assert now sits inside '__deleted'.
+        // There is nothing there for an Annotation to describe, so skip the gather and the merge
+        // and hand the record back exactly as it is stored.  Both counts report zero.
+        const deleted = utils.isDeleted(match)
         const targetId = match["@id"] ?? match.id
         // A record minted with a Slug also resolves at '/id/', so an Annotation may
         // target it by that URI instead of by its '@id'.
         const slug = match.__rerum?.slug
         const lastSlash = targetId?.lastIndexOf("/") ?? -1
         const slugTargetId = slug && lastSlash !== -1 ? targetId.slice(0, lastSlash + 1) + slug : undefined
-        const annos = await findLeafAnnotationsFor([targetId, slugTargetId], filters)
+        const annos = deleted ? [] : await findLeafAnnotationsFor([targetId, slugTargetId], filters)
         // Every leaf Annotation matching the filter is gathered.  This is the count.
         res.set('Annotations-Gathered', String(annos.length))
-        // Read each Annotation once.  These assertions are both the merged count and the merge itself.
         // How many of the Annotations contribute an assertion.  May be less than annotations gathered.
         const merged = annos.map(anno => assertionsFrom(anno)).filter(assertions => assertions.length > 0)
         res.set('Annotations-Merged', String(merged.length))
-        let expanded = applyExpansionAnnotations(match, merged)
+        let expanded = deleted ? match : applyExpansionAnnotations(match, merged)
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate")
diff --git a/controllers/utils.js b/controllers/utils.js
index e36ca069..e37830b7 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -76,7 +76,6 @@ const idNegotiation = function (resBody) {
     if(!resBody["@context"]) return resBody
     // Only this path hands back the body it was given, so it is the only one that needs a copy of it.
     if(!_contextid(resBody["@context"])) return structuredClone(resBody)
-    // The '@id' to 'id' path builds its result from scratch instead, and never reads a copy.
     const context = { "@context": resBody["@context"] }
     delete resBody["@id"]
     delete resBody["@context"]
diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml
index c08ad029..b5464689 100644
--- a/openapi/contracts/core-provider.openapi.yaml
+++ b/openapi/contracts/core-provider.openapi.yaml
@@ -72,7 +72,8 @@ paths:
         - $ref: '#/components/parameters/ExpansionCreator'
       responses:
         '200':
-          description: Expanded object payload
+          description: >-
+            Expanded object payload. A deleted record is returned as stored, with no expansion.
           headers:
             Annotations-Gathered:
               $ref: '#/components/headers/AnnotationsGathered'
@@ -101,7 +102,8 @@ paths:
         - $ref: '#/components/parameters/ExpansionCreator'
       responses:
         '200':
-          description: Expanded object headers
+          description: >-
+            Expanded object headers. A deleted record is returned as stored, with no expansion.
           headers:
             Annotations-Gathered:
               $ref: '#/components/headers/AnnotationsGathered'
@@ -130,7 +132,8 @@ paths:
               $ref: '#/components/schemas/GenericObject'
       responses:
         '200':
-          description: Expanded object payload
+          description: >-
+            Expanded object payload. A deleted record is returned as stored, with no expansion.
           headers:
             Annotations-Gathered:
               $ref: '#/components/headers/AnnotationsGathered'
diff --git a/public/API.html b/public/API.html
index 8b6892f0..c92b15eb 100644
--- a/public/API.html
+++ b/public/API.html
@@ -1721,6 +1721,15 @@ 
    Entity Expansion
    
             __proto__ contributes nothing. The expanded record
             answers to the same URI as the record you asked for.
         
    
+        
    
+            A deleted record is never expanded. Whatever it used to assert is inside its
+            __deleted snapshot, so there is nothing for an
+            Annotation to describe. The record comes back exactly as
+            GET /id/_id
+            returns it, and both count headers report
+            0. Annotations targeting the record are left
+            alone and are still retrievable on their own.
+        
    
         
    Expansion response headers
    
         
    
             Neither form of the request is paged. Every Annotation matching the filters is gathered before the record is
diff --git a/utils.js b/utils.js
index 7ee4cd7d..48cb78fd 100644
--- a/utils.js
+++ b/utils.js
@@ -28,7 +28,10 @@ isReleased        —always ""
 
 const configureRerumOptions = function(generator, received, update, extUpdate){
     let configuredObject = structuredClone(received)
-    let received_options = received.__rerum ? structuredClone(received.__rerum) : {}
+    //'__rerum' is a system property.  The only caller that hands us a trustworthy one is the update
+    //path, where 'received' is a record the database just gave us.  Every other caller is passing a
+    //request body, so whatever it put under '__rerum' is ignored outright rather than read.
+    let received_options = update && received.__rerum ? structuredClone(received.__rerum) : {}
     let history = {}
     let releases = {}
     let rerumOptions = {}
@@ -43,17 +46,20 @@ const configureRerumOptions = function(generator, received, update, extUpdate){
     }
     else{
         //We are either updating an existing RERUM object or creating a new one.
+        //'history' and 'releases' are always the objects built here.  A received __rerum is read for
+        //the two values below and nothing else, so no other property a caller put inside them can
+        //ride along into the stored record.
         if(received_options.hasOwnProperty("history")){
-            history = received_options.history
+            const received_history = received_options.history
             if(update){
                 //This means we are configuring from the update action and we have passed in a clone of the originating object (with its @id) that contained a __rerum.history
-                if(history.prime === "root"){
+                if(received_history.prime === "root"){
                     //Hitting this case means we are updating from the prime object, so we can't pass "root" on as the prime value
                     history_prime = received["@id"] ?? received.id ?? ""
                 }
                 else{
                     //Hitting this means we are updating an object that already knows its prime, so we can pass on the prime value
-                    history_prime = history.prime
+                    history_prime = received_history.prime
                 }
                 //Either way, we know the previous value shold be the @id of the object received here. 
                 history_previous = received["@id"] ?? received.id ?? ""
@@ -70,8 +76,7 @@ const configureRerumOptions = function(generator, received, update, extUpdate){
             history_previous = ""
         }
         if(received_options.hasOwnProperty("releases")){
-            releases = received_options.releases
-            releases_previous = releases.previous
+            releases_previous = received_options.releases.previous
         }
         else{
             releases_previous = ""         

From c43e6265b03c5ade85e34870290277871063a4ea Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 16:10:08 -0500
Subject: [PATCH 29/48] Changes from testing and review.

---
 controllers/crud.js  | 28 +++++++++++++++-------------
 controllers/gog.js   |  2 +-
 controllers/utils.js |  5 ++---
 public/API.html      | 17 +++++++++++++----
 4 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index 2d878178..a16b3aaf 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -127,18 +127,24 @@ const id = async function (req, res, next) {
     }
 }
 
+/**
+ * The expand job always constrains the Annotations it gathers to the leaf versions, to the
+ * Annotation types, and to the entity in the request URI.  A client cannot influence those, so
+ * these keys are dropped from a supplied filter body by exact name or dotted prefix.
+ */
+const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"]
+
+/**
+ * The Annotation body types whose value is kept whole rather than read as a single assertion.
+ */
+const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www.w3.org/ns/oa#TextualBody"])
+
 /**
  * Reduce a supplied POST body to the literal MongoDB filter keys the expand job will honor.
  * @param supplied The parsed JSON request body.
  * @return An object of filter keys, minus the ones this endpoint owns.
  */
 function sanitizeExpansionFilters(supplied) {
-    /**
-     * The expand job always constrains the Annotations it gathers to the leaf versions, to the
-     * Annotation types, and to the entity in the request URI.  A client cannot influence those, so
-     * these keys are dropped from a supplied filter body by exact name or dotted prefix.
-     */
-    const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"]
     const filters = {}
     for (const [key, value] of Object.entries(supplied)) {
         if (RESERVED_FILTER_KEYS.some(reserved => key === reserved || key.startsWith(`${reserved}.`))) continue
@@ -165,10 +171,6 @@ function sanitizeExpansionFilters(supplied) {
  * @return An Array of [key, value] pairs to merge onto the entity.
  */
 function assertionsFrom(anno) {
-    /**
-     * The Annotation body types whose value is kept whole rather than read as a single assertion.
-     */
-    const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www.w3.org/ns/oa#TextualBody"])
     const assertions = []
     if (typeof anno.bodyValue === "string") assertions.push(["bodyValue", anno.bodyValue])
     // In JSON-LD a one-element Array and the bare value are the same body, so unwrap it first.  The
@@ -228,7 +230,7 @@ function applyExpansionAnnotations(primitiveEntity, annoAssertions) {
  * */
 const idExpanded = async function (req, res, next) {
     res.set("Content-Type", "application/json; charset=utf-8")
-    const id = req.params["_id"]
+    const requestedId = req.params["_id"]
     const isPost = req.method === "POST"
     let filters = {}
     if (isPost) {
@@ -249,10 +251,10 @@ const idExpanded = async function (req, res, next) {
         if (typeof req.query.creator === "string" && req.query.creator) filters.creator = req.query.creator
     }
     try {
-        const match = await db.findOne({"$or": [{"_id": id}, {"__rerum.slug": id}]})
+        const match = await db.findOne({"$or": [{"_id": requestedId}, {"__rerum.slug": requestedId}]})
         if (!match) {
             const err = {
-                "message": `No RERUM object with id '${id}'`,
+                "message": `No RERUM object with id '${requestedId}'`,
                 "status": 404
             }
             return next(utils.createExpressError(err))
diff --git a/controllers/gog.js b/controllers/gog.js
index 2289ecb9..b6bdd39e 100644
--- a/controllers/gog.js
+++ b/controllers/gog.js
@@ -391,7 +391,7 @@ const expandedId = async function (req, res, next) {
             return next(utils.createExpressError(err))
         }
         res.set(utils.configureWebAnnoHeadersFor(match))
-        let expanded = await expand(match, generator)
+        let expanded = utils.isDeleted(match) ? match : await expand(match, generator)
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         res.set("Cache-Control", "max-age=86400, must-revalidate")
diff --git a/controllers/utils.js b/controllers/utils.js
index e37830b7..ea879bcf 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -125,10 +125,9 @@ const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "targ
 
 /**
  * Identity and system properties an Annotation body must never overwrite when its assertions are
- * merged onto an entity.  '__proto__' is not data -- emitting it would hand a prototype pollution
- * vector to every client that parses the response.
+ * merged onto an entity.
  */
-const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "@context", "__proto__"])
+const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "__proto__"])
 
 /**
  * Escape the RegExp metacharacters in a literal so it can be embedded in a pattern and match only
diff --git a/public/API.html b/public/API.html
index c92b15eb..23192691 100644
--- a/public/API.html
+++ b/public/API.html
@@ -186,12 +186,14 @@ 
    Expanded record by id
    
                 properties.
         
         
    
-            Annotation objects must use a target property to be gathered, and a body or bodyValue property to contribute.  See Entity Expansion for more information.
+            Annotation objects must carry a type or @type naming them an Annotation and a target property to be gathered, and a body or bodyValue property to contribute.  An object that targets the record but is not typed as an Annotation is not gathered.  See Entity Expansion for more information.
         
    
         
    
             Supply ?generator and
-            ?creator at most once each. A parameter supplied
-            more than once is not applied, and the response is the same unfiltered expansion.
+            ?creator at most once each, as a plain string. A
+            parameter supplied more than once, or supplied in a bracketed object form such as
+            ?generator[key]=value, is dropped rather than
+            applied. The other parameter still applies if it was supplied correctly.
         
    
         
    
             The response reports how much of the record's Annotation set went into it through the
@@ -1715,12 +1717,19 @@ 
    Entity Expansion
    
             asserting @id,
             id,
             _id,
-            @context,
             __rerum,
             __deleted, or
             __proto__ contributes nothing. The expanded record
             answers to the same URI as the record you asked for.
         
    
+        
    
+            Linked Data keywords that describe the record rather than identify it are not held back. An Annotation
+            asserting type,
+            @type, or
+            @context contributes it like any other property,
+            which collects the record's own value and the asserted one into an Array by the rule above. Saying what an
+            entity is, is the job of an Annotation.
+        
    
         
    
             A deleted record is never expanded. Whatever it used to assert is inside its
             __deleted snapshot, so there is nothing for an

From 96e794174613a246944c81dd5228569ba3337744 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 16:56:32 -0500
Subject: [PATCH 30/48] Linear big o instead of exponential.  Do id/context
 negotiation before expanding the entity.

---
 controllers/crud.js  | 15 +++++++++++----
 controllers/utils.js | 45 ++++++++++++++++++++------------------------
 2 files changed, 31 insertions(+), 29 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index a16b3aaf..54ffee6f 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -270,19 +270,26 @@ const idExpanded = async function (req, res, next) {
         const slug = match.__rerum?.slug
         const lastSlash = targetId?.lastIndexOf("/") ?? -1
         const slugTargetId = slug && lastSlash !== -1 ? targetId.slice(0, lastSlash + 1) + slug : undefined
+        // Read off the record while it is still whole.  idNegotiation() below alters it in place.
+        const currentVersion = match.__rerum?.isOverwritten ?? ""
         const annos = deleted ? [] : await findLeafAnnotationsFor([targetId, slugTargetId], filters)
         // Every leaf Annotation matching the filter is gathered.  This is the count.
         res.set('Annotations-Gathered', String(annos.length))
         // How many of the Annotations contribute an assertion.  May be less than annotations gathered.
         const merged = annos.map(anno => assertionsFrom(anno)).filter(assertions => assertions.length > 0)
         res.set('Annotations-Merged', String(merged.length))
-        let expanded = deleted ? match : applyExpansionAnnotations(match, merged)
-        expanded = idNegotiation(expanded)
+        // Negotiate the identity form from the record's own '@context', before an Annotation can
+        // contribute one.  An Annotation may assert '@context' like any other property, but it must
+        // not be able to change which property the record answers to.  Reading it off the merged
+        // object would let a contributed '@context' delete the record's '@id' and mint an 'id'.
+        const negotiated = idNegotiation(match)
+        const identity = _contextid(negotiated["@context"]) ? negotiated.id : negotiated["@id"]
+        const expanded = deleted ? negotiated : applyExpansionAnnotations(negotiated, merged)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate")
         // Include current version for optimistic locking
-        res.set('Current-Overwritten-Version', match.__rerum?.isOverwritten ?? "")
-        res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"])
+        res.set('Current-Overwritten-Version', currentVersion)
+        res.location(identity)
         res.json(expanded)
     } catch (error) {
         return next(utils.createExpressError(error))
diff --git a/controllers/utils.js b/controllers/utils.js
index ea879bcf..87e4e797 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -73,8 +73,9 @@ const idNegotiation = function (resBody) {
     if(!resBody) return
     const _id = resBody._id
     delete resBody._id
+    // A record with no '@context' needs no negotiation, so it is handed straight back.  The two
+    // paths below both produce a new object rather than the one they were given.
     if(!resBody["@context"]) return resBody
-    // Only this path hands back the body it was given, so it is the only one that needs a copy of it.
     if(!_contextid(resBody["@context"])) return structuredClone(resBody)
     const context = { "@context": resBody["@context"] }
     delete resBody["@id"]
@@ -150,15 +151,15 @@ function escapeRegex(literal) {
  *   - target: {'source':{'id':'uri'}}                           a SpecificResource with an embedded source
  *   - target: 'uri#xywh=0,0,100,100'                            a fragment of the resource
  * and the likely Annotation type formats
- *   - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"}
- *   - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
+ *   - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http(s)://www.w3.org/ns/oa#Annotation"}
+ *   - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http(s)://www.w3.org/ns/oa#Annotation"}
  *
  * Only 'target' and 'body' are read, whatever the type spelling says.
  *
  * Any entity can answer to more than one URI.  A record minted with a Slug resolves at both
  * '/id/<_id>' and '/id/', and an Annotation may legitimately target it by
- * either one.  Every match is gathered.  This walks the result set in batches of EXPANSION_BATCH_SIZE
- * until the database has no more to give.
+ * either one.  Every match is gathered, in a single cursor the driver pages in strides of
+ * EXPANSION_BATCH_SIZE.  Result order is whatever the query plan yields and is not guaranteed.
  *
  * @param targetIds The '@id' or 'id' URI of the entity being expanded, or an Array of the URIs it
  * is known by when it answers to more than one.
@@ -171,15 +172,17 @@ const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
     // Nothing to target is nothing to gather.  An empty '$or' is a MongoDB error, not an empty result.
     if (targetURIs.length === 0) return []
     // '$and' is always present so the target, type, and filter conditions can push into it.
-    // RERUM mints every '_id' as a hex string.  The batch resume below pages on '_id'.
     const queryObj = {
         "__rerum.history.next": { $exists: true, $size: 0 },
-        "_id": { $type: "string" },
         "$and": []
     }
+    // 'http://www.w3.org/ns/oa#' is the canonical namespace, but a serializer may spell it 'https'
+    // the same way RERUM has minted its own URIs under both.  Both spellings are matched.
     const annoTypeConditions = [
-        {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http://www.w3.org/ns/oa#Annotation"},
-        {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http://www.w3.org/ns/oa#Annotation"}
+        {"type": "Annotation"}, {"type": "oa:Annotation"},
+        {"type": "http://www.w3.org/ns/oa#Annotation"}, {"type": "https://www.w3.org/ns/oa#Annotation"},
+        {"@type": "Annotation"}, {"@type": "oa:Annotation"},
+        {"@type": "http://www.w3.org/ns/oa#Annotation"}, {"@type": "https://www.w3.org/ns/oa#Annotation"}
     ]
     // Every URI the entity answers to contributes its conditions to the same '$or'.
     const targetConditions = []
@@ -214,22 +217,14 @@ const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
         queryObj["$and"].push({ [key]: value })
     }
     const matches = []
-    let batch = []
-    let resumeAfter = null
-    do {
-        // The resume point joins the '_id' conditions rather than replacing them, so the '$type'
-        // constraint survives into every batch after the first.
-        const batchQuery = resumeAfter === null
-            ? queryObj
-            : { ...queryObj, "_id": { ...queryObj["_id"], $gt: resumeAfter } }
-        batch = await db.find(batchQuery).sort({ "_id": 1 }).limit(EXPANSION_BATCH_SIZE).toArray()
-        // Read the resume point before '_id' is dropped from the document.
-        if (batch.length > 0) resumeAfter = batch.at(-1)._id
-        for (const anno of batch) {
-            delete anno._id
-            matches.push(anno)
-        }
-    } while (batch.length === EXPANSION_BATCH_SIZE)
+    // One cursor, paged server-side by the driver.  Resuming on '_id' instead would need a sort the
+    // target indexes cannot provide -- the plan is an index union across the target keys -- so every
+    // batch would re-scan the whole match set behind a blocking in-memory sort, making a full gather
+    // quadratic.  The cursor transfers in the same stride and reads each document once.
+    for await (const anno of db.find(queryObj).batchSize(EXPANSION_BATCH_SIZE)) {
+        delete anno._id
+        matches.push(anno)
+    }
     return matches
 }
 

From 220c44e736174aaa15b499b1506e2c36f881b732 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 18:05:21 -0500
Subject: [PATCH 31/48] Changes during testing and review for expanding

---
 controllers/crud.js  | 25 +++++++++++++------------
 controllers/gog.js   | 12 ++++++++++--
 controllers/utils.js |  9 ++-------
 public/API.html      |  8 +++++---
 utils.js             |  6 ------
 5 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index 54ffee6f..f1ddb917 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -196,7 +196,8 @@ function assertionsFrom(anno) {
 /**
  * Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values.
  * When more than one current Annotation asserts the same key, or the entity already carries it,
- * the values collect into an Array.
+ * the values collect into an Array.  '@context' is the one key that collects in reverse, so the
+ * record's own context ends up last.
  * @param primitiveEntity The unexpanded entity.
  * @param annoAssertions An Array holding the [key, value] assertions read from each Annotation.
  * @return A new, expanded entity object.
@@ -213,7 +214,15 @@ function applyExpansionAnnotations(primitiveEntity, annoAssertions) {
                 continue
             }
             const existing = Array.isArray(expandedEntity[key]) ? expandedEntity[key] : [expandedEntity[key]]
-            expandedEntity[key] = Array.isArray(value) ? [...existing, ...value] : [...existing, value]
+            const contributed = Array.isArray(value) ? value : [value]
+            // '@context' collects in the opposite order.  The record's own context is what the
+            // identity form was negotiated from, so a contributed one goes in front of it and the
+            // record's own stays last -- JSON-LD applies an '@context' Array in order and lets a
+            // later entry override an earlier one, so last is where the record keeps the final say
+            // over every term it defines.  An Annotation extends the record's context, never redefines it.
+            expandedEntity[key] = key === "@context"
+                ? [...contributed, ...existing]
+                : [...existing, ...contributed]
         }
     }
     if (rerumProp !== undefined) expandedEntity.__rerum = rerumProp
@@ -259,14 +268,9 @@ const idExpanded = async function (req, res, next) {
             }
             return next(utils.createExpressError(err))
         }
-        res.set(utils.configureWebAnnoHeadersFor(match))
-        // A deleted record is a tombstone -- whatever it used to assert now sits inside '__deleted'.
-        // There is nothing there for an Annotation to describe, so skip the gather and the merge
-        // and hand the record back exactly as it is stored.  Both counts report zero.
         const deleted = utils.isDeleted(match)
         const targetId = match["@id"] ?? match.id
-        // A record minted with a Slug also resolves at '/id/', so an Annotation may
-        // target it by that URI instead of by its '@id'.
+        // an Annotation may target an entity by its Slug instead of by its '@id'.
         const slug = match.__rerum?.slug
         const lastSlash = targetId?.lastIndexOf("/") ?? -1
         const slugTargetId = slug && lastSlash !== -1 ? targetId.slice(0, lastSlash + 1) + slug : undefined
@@ -278,16 +282,13 @@ const idExpanded = async function (req, res, next) {
         // How many of the Annotations contribute an assertion.  May be less than annotations gathered.
         const merged = annos.map(anno => assertionsFrom(anno)).filter(assertions => assertions.length > 0)
         res.set('Annotations-Merged', String(merged.length))
-        // Negotiate the identity form from the record's own '@context', before an Annotation can
-        // contribute one.  An Annotation may assert '@context' like any other property, but it must
-        // not be able to change which property the record answers to.  Reading it off the merged
-        // object would let a contributed '@context' delete the record's '@id' and mint an 'id'.
         const negotiated = idNegotiation(match)
         const identity = _contextid(negotiated["@context"]) ? negotiated.id : negotiated["@id"]
         const expanded = deleted ? negotiated : applyExpansionAnnotations(negotiated, merged)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate")
         // Include current version for optimistic locking
+        res.set(utils.configureWebAnnoHeadersFor(expanded))
         res.set('Current-Overwritten-Version', currentVersion)
         res.location(identity)
         res.json(expanded)
diff --git a/controllers/gog.js b/controllers/gog.js
index b6bdd39e..6a2394a2 100644
--- a/controllers/gog.js
+++ b/controllers/gog.js
@@ -381,6 +381,14 @@ const expandedId = async function (req, res, next) {
             err = Object.assign(err, { message: `No RERUM object with id '${id}'`, status: 404 })
             return next(utils.createExpressError(err))
         }
+        if (utils.isDeleted(match)) {
+            res.set(utils.configureWebAnnoHeadersFor(match))
+            res.set("Cache-Control", "max-age=86400, must-revalidate")
+            res.set("Current-Overwritten-Version", "")
+            const tombstone = idNegotiation(match)
+            res.location(_contextid(tombstone["@context"]) ? tombstone.id : tombstone["@id"])
+            return res.json(tombstone)
+        }
         const generator = match.__rerum?.generatedBy
         const agentID = generator?.split("/").pop()
         if (!GOG_AGENTS.includes(agentID)) {
@@ -390,8 +398,8 @@ const expandedId = async function (req, res, next) {
             })
             return next(utils.createExpressError(err))
         }
-        res.set(utils.configureWebAnnoHeadersFor(match))
-        let expanded = utils.isDeleted(match) ? match : await expand(match, generator)
+        let expanded = await expand(match, generator)
+        res.set(utils.configureWebAnnoHeadersFor(expanded))
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         res.set("Cache-Control", "max-age=86400, must-revalidate")
diff --git a/controllers/utils.js b/controllers/utils.js
index 87e4e797..9b0588ee 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -30,9 +30,7 @@ function getPagination(query = {}, defaultLimit = 100) {
 /**
  * Check if a @context value contains a known @id-id mapping context
  *
- * @param contextInput A string URI, or an Array of them.  JSON-LD allows an '@context' Array to mix
- * URI strings with inline term definition objects, so a member that is not a string is skipped --
- * only a string can name one of the known contexts.
+ * @param contextInput A string URI, or an Array of them.
  * @return A boolean
  */
 function _contextid(contextInput) {
@@ -217,10 +215,7 @@ const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
         queryObj["$and"].push({ [key]: value })
     }
     const matches = []
-    // One cursor, paged server-side by the driver.  Resuming on '_id' instead would need a sort the
-    // target indexes cannot provide -- the plan is an index union across the target keys -- so every
-    // batch would re-scan the whole match set behind a blocking in-memory sort, making a full gather
-    // quadratic.  The cursor transfers in the same stride and reads each document once.
+    // One cursor, paged server-side by the driver. The cursor transfers in the same stride and reads each document once.
     for await (const anno of db.find(queryObj).batchSize(EXPANSION_BATCH_SIZE)) {
         delete anno._id
         matches.push(anno)
diff --git a/public/API.html b/public/API.html
index 23192691..064405a1 100644
--- a/public/API.html
+++ b/public/API.html
@@ -1726,9 +1726,11 @@ 
    Entity Expansion
    
             Linked Data keywords that describe the record rather than identify it are not held back. An Annotation
             asserting type,
             @type, or
-            @context contributes it like any other property,
-            which collects the record's own value and the asserted one into an Array by the rule above. Saying what an
-            entity is, is the job of an Annotation.
+            @context
+            contributes it like any other property, which collects the record's own value and the asserted one
+            into an Array. For @context the entity's
+            own @context will always appear as the last
+            entry or entries in the Array.
         
    
         
    
             A deleted record is never expanded. Whatever it used to assert is inside its
diff --git a/utils.js b/utils.js
index 48cb78fd..d499d07e 100644
--- a/utils.js
+++ b/utils.js
@@ -28,9 +28,6 @@ isReleased        —always ""
 
 const configureRerumOptions = function(generator, received, update, extUpdate){
     let configuredObject = structuredClone(received)
-    //'__rerum' is a system property.  The only caller that hands us a trustworthy one is the update
-    //path, where 'received' is a record the database just gave us.  Every other caller is passing a
-    //request body, so whatever it put under '__rerum' is ignored outright rather than read.
     let received_options = update && received.__rerum ? structuredClone(received.__rerum) : {}
     let history = {}
     let releases = {}
@@ -46,9 +43,6 @@ const configureRerumOptions = function(generator, received, update, extUpdate){
     }
     else{
         //We are either updating an existing RERUM object or creating a new one.
-        //'history' and 'releases' are always the objects built here.  A received __rerum is read for
-        //the two values below and nothing else, so no other property a caller put inside them can
-        //ride along into the stored record.
         if(received_options.hasOwnProperty("history")){
             const received_history = received_options.history
             if(update){

From 218f4a2d449e1d64650bc50bedfd938140b58b0f Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 18:11:42 -0500
Subject: [PATCH 32/48] undo these tests

---
 __tests__/utils.test.js | 54 -----------------------------------------
 1 file changed, 54 deletions(-)

diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js
index 3d862952..900ca7c3 100644
--- a/__tests__/utils.test.js
+++ b/__tests__/utils.test.js
@@ -49,60 +49,6 @@ describe('utils.js configureRerumOptions', () => {
     )
     assert.strictEqual(result.__rerum.generatedBy, 'https://store.rerum.io/v1/id/legitimate-agent')
   })
-
-  it('drops caller-supplied properties inside __rerum.history and __rerum.releases', () => {
-    const result = utils.configureRerumOptions(
-      'agent-uri',
-      {
-        a: 'b',
-        __rerum: {
-          history: { t: 'h', prime: 'forged', next: ['forged'] },
-          releases: { evil: true, previous: 'p', replaces: 'forged' }
-        }
-      },
-      false,
-      false
-    )
-    assert.deepStrictEqual(Object.keys(result.__rerum.history).sort(), ['next', 'previous', 'prime'])
-    assert.deepStrictEqual(Object.keys(result.__rerum.releases).sort(), ['next', 'previous', 'replaces'])
-    assert.deepStrictEqual(result.__rerum.history, { next: [], previous: '', prime: 'root' })
-    // A create reads nothing out of a supplied __rerum, so the release pointer is not seeded either.
-    assert.deepStrictEqual(result.__rerum.releases, { next: [], previous: '', replaces: '' })
-  })
-
-  it('does not mutate the __rerum it was handed', () => {
-    const received = { __rerum: { history: { t: 'h' }, releases: { evil: true } } }
-    utils.configureRerumOptions('agent-uri', received, false, false)
-    assert.deepStrictEqual(received.__rerum.history, { t: 'h' })
-    assert.deepStrictEqual(received.__rerum.releases, { evil: true })
-  })
-
-  it('still carries prime and releases.previous forward on an update', () => {
-    const fromPrime = utils.configureRerumOptions(
-      'agent-uri',
-      { '@id': 'https://store.rerum.io/v1/id/AAA', __rerum: { history: { prime: 'root', previous: '', next: [] } } },
-      true,
-      false
-    )
-    assert.strictEqual(fromPrime.__rerum.history.prime, 'https://store.rerum.io/v1/id/AAA')
-    assert.strictEqual(fromPrime.__rerum.history.previous, 'https://store.rerum.io/v1/id/AAA')
-
-    const fromChild = utils.configureRerumOptions(
-      'agent-uri',
-      {
-        '@id': 'https://store.rerum.io/v1/id/BBB',
-        __rerum: {
-          history: { prime: 'https://store.rerum.io/v1/id/AAA', previous: '', next: [] },
-          releases: { previous: 'https://store.rerum.io/v1/id/REL', next: [], replaces: '' }
-        }
-      },
-      true,
-      false
-    )
-    assert.strictEqual(fromChild.__rerum.history.prime, 'https://store.rerum.io/v1/id/AAA')
-    assert.strictEqual(fromChild.__rerum.history.previous, 'https://store.rerum.io/v1/id/BBB')
-    assert.strictEqual(fromChild.__rerum.releases.previous, 'https://store.rerum.io/v1/id/REL')
-  })
 })
 
 describe('controllers/utils.js generateSlugId', () => {

From 8f8d0683b2a5e4ce06f53ba54786a7f7fc02a70c Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 18:17:04 -0500
Subject: [PATCH 33/48] comment cleanup

---
 controllers/crud.js  | 6 +-----
 controllers/utils.js | 2 --
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index f1ddb917..db2e8dc9 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -215,11 +215,7 @@ function applyExpansionAnnotations(primitiveEntity, annoAssertions) {
             }
             const existing = Array.isArray(expandedEntity[key]) ? expandedEntity[key] : [expandedEntity[key]]
             const contributed = Array.isArray(value) ? value : [value]
-            // '@context' collects in the opposite order.  The record's own context is what the
-            // identity form was negotiated from, so a contributed one goes in front of it and the
-            // record's own stays last -- JSON-LD applies an '@context' Array in order and lets a
-            // later entry override an earlier one, so last is where the record keeps the final say
-            // over every term it defines.  An Annotation extends the record's context, never redefines it.
+            // '@context' collects in the opposite order.  The record's own context is last.
             expandedEntity[key] = key === "@context"
                 ? [...contributed, ...existing]
                 : [...existing, ...contributed]
diff --git a/controllers/utils.js b/controllers/utils.js
index 9b0588ee..53f49612 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -71,8 +71,6 @@ const idNegotiation = function (resBody) {
     if(!resBody) return
     const _id = resBody._id
     delete resBody._id
-    // A record with no '@context' needs no negotiation, so it is handed straight back.  The two
-    // paths below both produce a new object rather than the one they were given.
     if(!resBody["@context"]) return resBody
     if(!_contextid(resBody["@context"])) return structuredClone(resBody)
     const context = { "@context": resBody["@context"] }

From 7848f73a94c46aa830b0450e5ef406c6cf20dc72 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 18:40:03 -0500
Subject: [PATCH 34/48] Changes during testing and review

---
 controllers/crud.js | 3 ++-
 controllers/gog.js  | 3 +++
 utils.js            | 7 ++++---
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index db2e8dc9..fd264209 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -278,13 +278,14 @@ const idExpanded = async function (req, res, next) {
         // How many of the Annotations contribute an assertion.  May be less than annotations gathered.
         const merged = annos.map(anno => assertionsFrom(anno)).filter(assertions => assertions.length > 0)
         res.set('Annotations-Merged', String(merged.length))
+        // Negotiate before the merge, so a contributed '@context' cannot change which property the record answers to.
         const negotiated = idNegotiation(match)
         const identity = _contextid(negotiated["@context"]) ? negotiated.id : negotiated["@id"]
         const expanded = deleted ? negotiated : applyExpansionAnnotations(negotiated, merged)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate")
-        // Include current version for optimistic locking
         res.set(utils.configureWebAnnoHeadersFor(expanded))
+        // Include current version for optimistic locking
         res.set('Current-Overwritten-Version', currentVersion)
         res.location(identity)
         res.json(expanded)
diff --git a/controllers/gog.js b/controllers/gog.js
index 6a2394a2..0916cb27 100644
--- a/controllers/gog.js
+++ b/controllers/gog.js
@@ -381,6 +381,7 @@ const expandedId = async function (req, res, next) {
             err = Object.assign(err, { message: `No RERUM object with id '${id}'`, status: 404 })
             return next(utils.createExpressError(err))
         }
+        // Deleted objects don't have a generator to match on, just return the tombstone no matter what.
         if (utils.isDeleted(match)) {
             res.set(utils.configureWebAnnoHeadersFor(match))
             res.set("Cache-Control", "max-age=86400, must-revalidate")
@@ -400,6 +401,8 @@ const expandedId = async function (req, res, next) {
         }
         let expanded = await expand(match, generator)
         res.set(utils.configureWebAnnoHeadersFor(expanded))
+        // Safe to negotiate after the merge here.  expand() wraps every assertion in a
+        // valueObject, so a contributed '@context' is a non-string that _contextid() skips.
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         res.set("Cache-Control", "max-age=86400, must-revalidate")
diff --git a/utils.js b/utils.js
index d499d07e..9a02f6c3 100644
--- a/utils.js
+++ b/utils.js
@@ -178,7 +178,8 @@ const configureLDHeadersFor = function(obj){
  */ 
 const isContainerType = function(obj){
     let answer = false
-    let typestring = obj["@type"] ?? obj.type ?? ""
+    const typeValue = obj["@type"] ?? obj.type ?? ""
+    const typestrings = (Array.isArray(typeValue) ? typeValue : [typeValue]).filter(t => typeof t === "string")
     const knownContainerTypes = [
         "ItemList",
         "AnnotationPage",
@@ -191,9 +192,9 @@ const isContainerType = function(obj){
         "Set",
         "Collection"
     ]
-    for(const t of knownContainerTypes){
+    for(const typestring of typestrings){
         //Dang those pesky prefixes...circumventing exact match for now
-        if(typestring.includes(t)){
+        if(knownContainerTypes.some(t => typestring.includes(t))){
             answer = true
             break
         }

From 0c32ddc05ab55327e107852cdc01e6bfaf56ab88 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 19:10:12 -0500
Subject: [PATCH 35/48] Changes during testing and review

---
 controllers/crud.js  | 17 +++++++++--------
 controllers/gog.js   |  4 ++--
 controllers/utils.js | 16 +++++++++++-----
 public/API.html      | 29 ++++++++++++++++++-----------
 4 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index fd264209..7a565116 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -136,8 +136,13 @@ const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"]
 
 /**
  * The Annotation body types whose value is kept whole rather than read as a single assertion.
+ * 'http://www.w3.org/ns/oa#' is the canonical namespace, but a serializer may spell it 'https', so
+ * both are matched here the same way they are in the Annotation type conditions.
  */
-const TEXTUAL_BODY_TYPES = new Set(["TextualBody", "oa:TextualBody", "http://www.w3.org/ns/oa#TextualBody"])
+const TEXTUAL_BODY_TYPES = new Set([
+    "TextualBody", "oa:TextualBody",
+    "http://www.w3.org/ns/oa#TextualBody", "https://www.w3.org/ns/oa#TextualBody"
+])
 
 /**
  * Reduce a supplied POST body to the literal MongoDB filter keys the expand job will honor.
@@ -196,8 +201,7 @@ function assertionsFrom(anno) {
 /**
  * Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values.
  * When more than one current Annotation asserts the same key, or the entity already carries it,
- * the values collect into an Array.  '@context' is the one key that collects in reverse, so the
- * record's own context ends up last.
+ * the values collect into an Array, the record's own value first.
  * @param primitiveEntity The unexpanded entity.
  * @param annoAssertions An Array holding the [key, value] assertions read from each Annotation.
  * @return A new, expanded entity object.
@@ -215,10 +219,7 @@ function applyExpansionAnnotations(primitiveEntity, annoAssertions) {
             }
             const existing = Array.isArray(expandedEntity[key]) ? expandedEntity[key] : [expandedEntity[key]]
             const contributed = Array.isArray(value) ? value : [value]
-            // '@context' collects in the opposite order.  The record's own context is last.
-            expandedEntity[key] = key === "@context"
-                ? [...contributed, ...existing]
-                : [...existing, ...contributed]
+            expandedEntity[key] = [...existing, ...contributed]
         }
     }
     if (rerumProp !== undefined) expandedEntity.__rerum = rerumProp
@@ -278,7 +279,7 @@ const idExpanded = async function (req, res, next) {
         // How many of the Annotations contribute an assertion.  May be less than annotations gathered.
         const merged = annos.map(anno => assertionsFrom(anno)).filter(assertions => assertions.length > 0)
         res.set('Annotations-Merged', String(merged.length))
-        // Negotiate before the merge, so a contributed '@context' cannot change which property the record answers to.
+        // Negotiate first, so identity is settled from the record's own '@context' before anything is merged.
         const negotiated = idNegotiation(match)
         const identity = _contextid(negotiated["@context"]) ? negotiated.id : negotiated["@id"]
         const expanded = deleted ? negotiated : applyExpansionAnnotations(negotiated, merged)
diff --git a/controllers/gog.js b/controllers/gog.js
index 0916cb27..19f1d9bc 100644
--- a/controllers/gog.js
+++ b/controllers/gog.js
@@ -401,8 +401,8 @@ const expandedId = async function (req, res, next) {
         }
         let expanded = await expand(match, generator)
         res.set(utils.configureWebAnnoHeadersFor(expanded))
-        // Safe to negotiate after the merge here.  expand() wraps every assertion in a
-        // valueObject, so a contributed '@context' is a non-string that _contextid() skips.
+        // Safe to negotiate after the merge here.  '@context' is a protected key, so no Annotation
+        // can contribute one, and expand() wraps every assertion it does merge in a valueObject.
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         res.set("Cache-Control", "max-age=86400, must-revalidate")
diff --git a/controllers/utils.js b/controllers/utils.js
index 53f49612..5304eb3b 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -121,10 +121,15 @@ const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"])
 const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"]
 
 /**
- * Identity and system properties an Annotation body must never overwrite when its assertions are
- * merged onto an entity.
+ * Identity, system, and processing properties an Annotation body must never overwrite when its
+ * assertions are merged onto an entity.
+ * '@context' is here because it is a processing directive rather than an assertion about the
+ * entity.  A merged-in context is cumulative in JSON-LD -- a term, '@vocab', or '@base' that the
+ * record's own context does not redefine still applies no matter where it sits in the Array -- and
+ * it can flip _contextid(), which answers true for a known context anywhere in an Array.  A body
+ * whose only key is '@context' asserts nothing, so holding it back costs no expressiveness.
  */
-const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "__proto__"])
+const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "__proto__", "@context"])
 
 /**
  * Escape the RegExp metacharacters in a literal so it can be embedded in a pattern and match only
@@ -184,8 +189,9 @@ const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
     const targetConditions = []
     for (const targetURI of targetURIs) {
         if (!/^https?:\/\//.test(targetURI)) {
-            // Not a URI, so there is no http/https spelling or fragment of it to anticipate.
-            targetConditions.push({ "target": targetURI })
+            // Not a URI, so there is no http/https spelling or fragment of it to anticipate, but it
+            // can still sit under any of the target keys.
+            for (const targetKey of TARGET_KEYS) targetConditions.push({ [targetKey]: targetURI })
             continue
         }
         // Hanging a fragment off the URI is the other W3C way to target part of a resource rather
diff --git a/public/API.html b/public/API.html
index 064405a1..16cf25cc 100644
--- a/public/API.html
+++ b/public/API.html
@@ -1713,24 +1713,31 @@ 
    Entity Expansion
    
             record said and what each Annotation added.
         
    
         
    
-            A record's identity and system properties are never overwritten by an Annotation. An Annotation body
-            asserting @id,
+            A record's identity, system, and processing properties are never overwritten by an Annotation. An
+            Annotation body asserting @id,
             id,
             _id,
             __rerum,
-            __deleted, or
-            __proto__ contributes nothing. The expanded record
-            answers to the same URI as the record you asked for.
+            __deleted,
+            __proto__, or
+            @context contributes nothing. The expanded record
+            answers to the same URI as the record you asked for, and reads under the same
+            @context.
+        
    
+        
    
+            @context is held back because it is a processing
+            directive rather than something an Annotation asserts about the record. A body whose only key is
+            @context asserts nothing, and a merged-in context
+            would be cumulative—a term, @vocab, or
+            @base that the record's own
+            @context does not redefine would still apply.
         
    
         
    
             Linked Data keywords that describe the record rather than identify it are not held back. An Annotation
-            asserting type,
-            @type, or
-            @context
+            asserting type or
+            @type
             contributes it like any other property, which collects the record's own value and the asserted one
-            into an Array. For @context the entity's
-            own @context will always appear as the last
-            entry or entries in the Array.
+            into an Array.
         
    
         
    
             A deleted record is never expanded. Whatever it used to assert is inside its

From 683102aba488732db28a524645bbdaf2e7ac4973 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 19:17:42 -0500
Subject: [PATCH 36/48] Changes during testing and review

---
 controllers/crud.js  | 2 --
 controllers/utils.js | 7 -------
 public/API.html      | 8 --------
 3 files changed, 17 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index 7a565116..a25f159f 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -136,8 +136,6 @@ const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"]
 
 /**
  * The Annotation body types whose value is kept whole rather than read as a single assertion.
- * 'http://www.w3.org/ns/oa#' is the canonical namespace, but a serializer may spell it 'https', so
- * both are matched here the same way they are in the Annotation type conditions.
  */
 const TEXTUAL_BODY_TYPES = new Set([
     "TextualBody", "oa:TextualBody",
diff --git a/controllers/utils.js b/controllers/utils.js
index 5304eb3b..7f31c455 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -123,11 +123,6 @@ const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "targ
 /**
  * Identity, system, and processing properties an Annotation body must never overwrite when its
  * assertions are merged onto an entity.
- * '@context' is here because it is a processing directive rather than an assertion about the
- * entity.  A merged-in context is cumulative in JSON-LD -- a term, '@vocab', or '@base' that the
- * record's own context does not redefine still applies no matter where it sits in the Array -- and
- * it can flip _contextid(), which answers true for a known context anywhere in an Array.  A body
- * whose only key is '@context' asserts nothing, so holding it back costs no expressiveness.
  */
 const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "__proto__", "@context"])
 
@@ -177,8 +172,6 @@ const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
         "__rerum.history.next": { $exists: true, $size: 0 },
         "$and": []
     }
-    // 'http://www.w3.org/ns/oa#' is the canonical namespace, but a serializer may spell it 'https'
-    // the same way RERUM has minted its own URIs under both.  Both spellings are matched.
     const annoTypeConditions = [
         {"type": "Annotation"}, {"type": "oa:Annotation"},
         {"type": "http://www.w3.org/ns/oa#Annotation"}, {"type": "https://www.w3.org/ns/oa#Annotation"},
diff --git a/public/API.html b/public/API.html
index 16cf25cc..c20aa99c 100644
--- a/public/API.html
+++ b/public/API.html
@@ -1724,14 +1724,6 @@ 
    Entity Expansion
    
             answers to the same URI as the record you asked for, and reads under the same
             @context.
         
    
-        
    
-            @context is held back because it is a processing
-            directive rather than something an Annotation asserts about the record. A body whose only key is
-            @context asserts nothing, and a merged-in context
-            would be cumulative—a term, @vocab, or
-            @base that the record's own
-            @context does not redefine would still apply.
-        
    
         
    
             Linked Data keywords that describe the record rather than identify it are not held back. An Annotation
             asserting type or

From ec43208186280d0579f981901f91f6c71f50f635 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 20:40:05 -0500
Subject: [PATCH 37/48] Changes during testing and review

---
 controllers/crud.js  |  5 ++++-
 controllers/gog.js   |  6 ++----
 controllers/utils.js | 15 ++++++++++++---
 public/API.html      |  5 +++--
 4 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index a25f159f..91d9887b 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -283,7 +283,10 @@ const idExpanded = async function (req, res, next) {
         const expanded = deleted ? negotiated : applyExpansionAnnotations(negotiated, merged)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate")
-        res.set(utils.configureWebAnnoHeadersFor(expanded))
+        // Headers describe the stored record, so they match GET /v1/id/:_id for the same record.
+        // 'type' and '@type' are not protected keys, so an Annotation can contribute one to the
+        // payload.  It does not get to change what this URI advertises itself as.
+        res.set(utils.configureWebAnnoHeadersFor(negotiated))
         // Include current version for optimistic locking
         res.set('Current-Overwritten-Version', currentVersion)
         res.location(identity)
diff --git a/controllers/gog.js b/controllers/gog.js
index 19f1d9bc..fe6e1ce6 100644
--- a/controllers/gog.js
+++ b/controllers/gog.js
@@ -394,15 +394,13 @@ const expandedId = async function (req, res, next) {
         const agentID = generator?.split("/").pop()
         if (!GOG_AGENTS.includes(agentID)) {
             err = Object.assign(err, {
-                message: `This request can only be made for Gallery of Glosses generated data.`,
-                status: 403
+                message: `No Gallery of Glosses record with id '${id}'.  This URI serves GoG generated data only.`,
+                status: 404
             })
             return next(utils.createExpressError(err))
         }
         let expanded = await expand(match, generator)
         res.set(utils.configureWebAnnoHeadersFor(expanded))
-        // Safe to negotiate after the merge here.  '@context' is a protected key, so no Annotation
-        // can contribute one, and expand() wraps every assertion it does merge in a valueObject.
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         res.set("Cache-Control", "max-age=86400, must-revalidate")
diff --git a/controllers/utils.js b/controllers/utils.js
index 7f31c455..d6ebc2fb 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -155,12 +155,13 @@ function escapeRegex(literal) {
  * Any entity can answer to more than one URI.  A record minted with a Slug resolves at both
  * '/id/<_id>' and '/id/', and an Annotation may legitimately target it by
  * either one.  Every match is gathered, in a single cursor the driver pages in strides of
- * EXPANSION_BATCH_SIZE.  Result order is whatever the query plan yields and is not guaranteed.
+ * EXPANSION_BATCH_SIZE.  The query plan yields no order, so the result is sorted by '_id' before it
+ * is returned.  That is roughly Annotation creation order, and it makes the expansion reproducible.
  *
  * @param targetIds The '@id' or 'id' URI of the entity being expanded, or an Array of the URIs it
  * is known by when it answers to more than one.
  * @param filters Literal MongoDB filter keys to AND into the query.
- * @return An Array of every matching Annotation document, with '_id' removed.
+ * @return An Array of every matching Annotation document sorted by '_id', with '_id' removed.
  */
 const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
     const EXPANSION_BATCH_SIZE = 200
@@ -214,9 +215,17 @@ const findLeafAnnotationsFor = async function (targetIds, filters = {}) {
     const matches = []
     // One cursor, paged server-side by the driver. The cursor transfers in the same stride and reads each document once.
     for await (const anno of db.find(queryObj).batchSize(EXPANSION_BATCH_SIZE)) {
-        delete anno._id
         matches.push(anno)
     }
+    // The query plan promises no order, so sort before '_id' is dropped.  The same data then always
+    // assembles into the same entity, which keeps the ETag stable and lets a revalidation answer 304.
+    matches.sort((a, b) => {
+        const left = String(a._id)
+        const right = String(b._id)
+        if (left < right) return -1
+        return left > right ? 1 : 0
+    })
+    for (const anno of matches) delete anno._id
     return matches
 }
 
diff --git a/public/API.html b/public/API.html
index c20aa99c..3cd1a6a6 100644
--- a/public/API.html
+++ b/public/API.html
@@ -1702,8 +1702,9 @@ 
    Entity Expansion
    
             an Array. Two Annotations asserting
             ["red", "blue"] and
             ["black", "white"] produce
-            ["red", "blue", "black", "white"]. 
-            Array order is not guaranteed.
+            ["red", "blue", "black", "white"].
+            The record's own value comes first, then the Annotations in the order they were created. Asking twice for a
+            record whose Annotations have not changed returns the same assembly both times.
         
    
         
    
             This holds for a property the record already carries as an Array too, so an Annotation asserting something

From 5eec897bf1ddc14c0885c86ace804a80cac0fcb6 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Wed, 19 Aug 2026 20:47:46 -0500
Subject: [PATCH 38/48] Changes during testing and review

---
 controllers/crud.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index 91d9887b..7bafdca0 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -284,8 +284,6 @@ const idExpanded = async function (req, res, next) {
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate")
         // Headers describe the stored record, so they match GET /v1/id/:_id for the same record.
-        // 'type' and '@type' are not protected keys, so an Annotation can contribute one to the
-        // payload.  It does not get to change what this URI advertises itself as.
         res.set(utils.configureWebAnnoHeadersFor(negotiated))
         // Include current version for optimistic locking
         res.set('Current-Overwritten-Version', currentVersion)

From a841d36964d28cfa1fba109b47151e3b179ed80b Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 09:10:29 -0500
Subject: [PATCH 39/48] Settled on functionality, time to write tests.

---
 controllers/gog.js |  3 ++-
 utils.js           | 23 ++++++++---------------
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/controllers/gog.js b/controllers/gog.js
index fe6e1ce6..c3005dc6 100644
--- a/controllers/gog.js
+++ b/controllers/gog.js
@@ -400,7 +400,8 @@ const expandedId = async function (req, res, next) {
             return next(utils.createExpressError(err))
         }
         let expanded = await expand(match, generator)
-        res.set(utils.configureWebAnnoHeadersFor(expanded))
+        // Headers describe the stored record, so they match GET /v1/id/:_id for the same record.
+        res.set(utils.configureWebAnnoHeadersFor(match))
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         res.set("Cache-Control", "max-age=86400, must-revalidate")
diff --git a/utils.js b/utils.js
index 9a02f6c3..9143fe82 100644
--- a/utils.js
+++ b/utils.js
@@ -45,24 +45,17 @@ const configureRerumOptions = function(generator, received, update, extUpdate){
         //We are either updating an existing RERUM object or creating a new one.
         if(received_options.hasOwnProperty("history")){
             const received_history = received_options.history
-            if(update){
-                //This means we are configuring from the update action and we have passed in a clone of the originating object (with its @id) that contained a __rerum.history
-                if(received_history.prime === "root"){
-                    //Hitting this case means we are updating from the prime object, so we can't pass "root" on as the prime value
-                    history_prime = received["@id"] ?? received.id ?? ""
-                }
-                else{
-                    //Hitting this means we are updating an object that already knows its prime, so we can pass on the prime value
-                    history_prime = received_history.prime
-                }
-                //Either way, we know the previous value shold be the @id of the object received here. 
-                history_previous = received["@id"] ?? received.id ?? ""
+            //This means we are configuring from the update action and we have passed in a clone of the originating object (with its @id) that contained a __rerum.history
+            if(received_history.prime === "root"){
+                //Hitting this case means we are updating from the prime object, so we can't pass "root" on as the prime value
+                history_prime = received["@id"] ?? received.id ?? ""
             }
             else{
-                //Hitting this means we are saving a new object and found that __rerum.history existed.  We don't trust it, act like it doesn't have it.
-                history_prime = "root"
-                history_previous = ""
+                //Hitting this means we are updating an object that already knows its prime, so we can pass on the prime value
+                history_prime = received_history.prime
             }
+            //Either way, we know the previous value shold be the @id of the object received here. 
+            history_previous = received["@id"] ?? received.id ?? ""
         }
         else{
             //Hitting this means we are are saving an object that did not have __rerum history.  This is normal   

From 0d38d52d871754f00605312a43afe10a23878464 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 09:27:29 -0500
Subject: [PATCH 40/48] More honest merged count

---
 controllers/crud.js         |  6 +++++-
 database/__mocks__/index.js | 20 +++++++++++++++++---
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/controllers/crud.js b/controllers/crud.js
index 7bafdca0..05b3343b 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -275,7 +275,11 @@ const idExpanded = async function (req, res, next) {
         // Every leaf Annotation matching the filter is gathered.  This is the count.
         res.set('Annotations-Gathered', String(annos.length))
         // How many of the Annotations contribute an assertion.  May be less than annotations gathered.
-        const merged = annos.map(anno => assertionsFrom(anno)).filter(assertions => assertions.length > 0)
+        // The count reports the Annotations that actually alter the entity.
+        const merged = annos
+            .map(anno => assertionsFrom(anno)
+            .filter(([key]) => !PROTECTED_EXPANSION_KEYS.has(key)))
+            .filter(assertions => assertions.length > 0)
         res.set('Annotations-Merged', String(merged.length))
         // Negotiate first, so identity is settled from the record's own '@context' before anything is merged.
         const negotiated = idNegotiation(match)
diff --git a/database/__mocks__/index.js b/database/__mocks__/index.js
index 51ec6a2c..886842ba 100644
--- a/database/__mocks__/index.js
+++ b/database/__mocks__/index.js
@@ -39,12 +39,26 @@ function createMockFunction(implementation = () => undefined) {
   return fn
 }
 
-function createCursor() {
-  return {
+/**
+ * A stand-in for the driver's FindCursor.  The chainable methods return the cursor the way the
+ * driver's do, and the cursor is async-iterable over whatever toArray() resolves to so a caller
+ * can read it either way.
+ *
+ * @param docs The documents this cursor yields.
+ * @return A cursor double.
+ */
+export function createCursor(docs = []) {
+  const cursor = {
     limit: createMockFunction(function () { return this }),
     skip: createMockFunction(function () { return this }),
-    toArray: createMockFunction(() => Promise.resolve([]))
+    batchSize: createMockFunction(function () { return this }),
+    toArray: createMockFunction(() => Promise.resolve(docs)),
+    async *[Symbol.asyncIterator]() {
+      // A cursor whose toArray was reset has no implementation left, so it iterates as empty.
+      for (const doc of await cursor.toArray() ?? []) yield doc
+    }
   }
+  return cursor
 }
 
 const defaultBulkWriteResponse = () => ({

From b26d42824ded4a78c96e250234c98562d5705992 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 09:30:02 -0500
Subject: [PATCH 41/48] consistency

---
 controllers/gog.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/controllers/gog.js b/controllers/gog.js
index c3005dc6..778658a7 100644
--- a/controllers/gog.js
+++ b/controllers/gog.js
@@ -400,11 +400,11 @@ const expandedId = async function (req, res, next) {
             return next(utils.createExpressError(err))
         }
         let expanded = await expand(match, generator)
-        // Headers describe the stored record, so they match GET /v1/id/:_id for the same record.
-        res.set(utils.configureWebAnnoHeadersFor(match))
         expanded = idNegotiation(expanded)
         // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
         res.set("Cache-Control", "max-age=86400, must-revalidate")
+        // Headers describe the stored record, so they match GET /v1/id/:_id for the same record.
+        res.set(utils.configureWebAnnoHeadersFor(match))
         // Include current version for optimistic locking
         res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "")
         res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"])

From d25045609daa641629fda60d4ad1394c92b0c299 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 12:25:52 -0500
Subject: [PATCH 42/48] First pass at tests

---
 __tests__/core_provider_contract.test.js |   5 +
 __tests__/routes_mounted.test.js         |   5 +-
 __tests__/utils.test.js                  | 191 ++++++++++++++-
 routes/__tests__/id.test.js              | 296 ++++++++++++++++++++++-
 routes/__tests__/route_wrappers.test.js  |  11 +
 5 files changed, 504 insertions(+), 4 deletions(-)

diff --git a/__tests__/core_provider_contract.test.js b/__tests__/core_provider_contract.test.js
index c73d1309..6b34a726 100644
--- a/__tests__/core_provider_contract.test.js
+++ b/__tests__/core_provider_contract.test.js
@@ -199,6 +199,11 @@ const requiredResponseCodes = {
   // 409 is reachable via slug conflict (utils.createExpressError maps code 11000 → 409).
   'PATCH /api/release/{id}': ['200', '400', '401', '403', '404', '409'],
   'GET /id/{id}': ['200', '404'],
+  // The expanded reads are covered in routes/__tests__/id.test.js: 404 on a miss, and a
+  // POST that reads filters from its body, so it also answers the body-related codes.
+  'GET /id/{id}/expanded': ['200', '404'],
+  'HEAD /id/{id}/expanded': ['200', '404'],
+  'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'],
   'GET /since/{id}': ['200', '404'],
   'GET /history/{id}': ['200', '404'],
   // HEAD parity tests in routes/__tests__/{id,since,history,query}.test.js assert 404 on miss;
diff --git a/__tests__/routes_mounted.test.js b/__tests__/routes_mounted.test.js
index aacb2094..3e40f9aa 100644
--- a/__tests__/routes_mounted.test.js
+++ b/__tests__/routes_mounted.test.js
@@ -26,7 +26,10 @@ const mountedApiRoutes = [
   { name: '/v1/api/delete/{id}', method: 'delete', path: '/v1/api/delete/test-mounted-id' },
   { name: '/v1/api/release/{id}', method: 'patch', path: '/v1/api/release/test-mounted-id' },
   { name: '/v1/api/search', method: 'post', path: '/v1/api/search', headers: { 'Content-Type': 'text/plain' }, body: 'mounted search' },
-  { name: '/v1/api/search/phrase', method: 'post', path: '/v1/api/search/phrase', headers: { 'Content-Type': 'text/plain' }, body: 'mounted phrase search' }
+  { name: '/v1/api/search/phrase', method: 'post', path: '/v1/api/search/phrase', headers: { 'Content-Type': 'text/plain' }, body: 'mounted phrase search' },
+  // A missing record answers 404 here, which an unmounted path would too.  PUT is the
+  // method this route rejects, so a non-404 proves the router is wired up.
+  { name: '/v1/id/{_id}/expanded', method: 'put', path: '/v1/id/test-mounted-id/expanded' }
 ]
 
 describe('Mounted route surface', () => {
diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js
index 900ca7c3..3cb20811 100644
--- a/__tests__/utils.test.js
+++ b/__tests__/utils.test.js
@@ -9,9 +9,10 @@ import {
   parseDocumentID,
   _contextid,
   idNegotiation,
-  getPagination
+  getPagination,
+  findLeafAnnotationsFor
 } from '../controllers/utils.js'
-import { db, resetMocks } from '../database/index.js'
+import { db, resetMocks, createCursor } from '../database/index.js'
 
 describe('utils.js auth gates', () => {
   it('isDeleted returns true only for objects with __deleted', () => {
@@ -49,6 +50,56 @@ describe('utils.js configureRerumOptions', () => {
     )
     assert.strictEqual(result.__rerum.generatedBy, 'https://store.rerum.io/v1/id/legitimate-agent')
   })
+
+  const AGENT = 'https://store.rerum.io/v1/id/legitimate-agent'
+  const RECEIVED_ID = 'https://store.rerum.io/v1/id/received-id'
+  const FORGED = {
+    history: { prime: 'https://store.rerum.io/v1/id/forged-prime', previous: 'https://store.rerum.io/v1/id/forged-previous', next: ['https://store.rerum.io/v1/id/forged-next'] },
+    releases: { previous: 'https://store.rerum.io/v1/id/forged-release', next: [], replaces: '' }
+  }
+
+  it('ignores a client-supplied __rerum when minting a new object', () => {
+    const created = utils.configureRerumOptions(AGENT, { '@id': RECEIVED_ID, __rerum: structuredClone(FORGED) }, false, false)
+    assert.strictEqual(created.__rerum.history.prime, 'root')
+    assert.strictEqual(created.__rerum.history.previous, '')
+    assert.deepStrictEqual(created.__rerum.history.next, [])
+    assert.strictEqual(created.__rerum.releases.previous, '')
+
+    // An external object imported through an update is also a root, but it remembers its external self.
+    const imported = utils.configureRerumOptions(AGENT, { '@id': 'https://elsewhere.example.org/thing', __rerum: structuredClone(FORGED) }, false, true)
+    assert.strictEqual(imported.__rerum.history.prime, 'root')
+    assert.strictEqual(imported.__rerum.history.previous, 'https://elsewhere.example.org/thing')
+    assert.strictEqual(imported.__rerum.releases.previous, '')
+  })
+
+  it('carries the version and release chain forward when updating', () => {
+    const fromRoot = utils.configureRerumOptions(
+      AGENT,
+      { '@id': RECEIVED_ID, __rerum: { history: { prime: 'root', previous: '', next: [] } } },
+      true,
+      false
+    )
+    assert.strictEqual(fromRoot.__rerum.history.prime, RECEIVED_ID, 'the root object cannot pass "root" on as the prime')
+    assert.strictEqual(fromRoot.__rerum.history.previous, RECEIVED_ID)
+
+    const PRIME = 'https://store.rerum.io/v1/id/prime-id'
+    const RELEASE = 'https://store.rerum.io/v1/id/released-id'
+    const fromDescendant = utils.configureRerumOptions(
+      AGENT,
+      {
+        '@id': RECEIVED_ID,
+        __rerum: {
+          history: { prime: PRIME, previous: 'https://store.rerum.io/v1/id/older-id', next: [] },
+          releases: { previous: RELEASE, next: [], replaces: '' }
+        }
+      },
+      true,
+      false
+    )
+    assert.strictEqual(fromDescendant.__rerum.history.prime, PRIME, 'an object that knows its prime passes it on')
+    assert.strictEqual(fromDescendant.__rerum.history.previous, RECEIVED_ID)
+    assert.strictEqual(fromDescendant.__rerum.releases.previous, RELEASE)
+  })
 })
 
 describe('controllers/utils.js generateSlugId', () => {
@@ -219,6 +270,14 @@ describe('controllers/utils.js _contextid', () => {
     assert.strictEqual(_contextid(123), false)
     assert.strictEqual(_contextid({}), false)
   })
+
+  it('skips non-string members of an array, such as an inline term definition', () => {
+    assert.strictEqual(
+      _contextid([{ '@vocab': 'http://example.org/terms#' }, 'http://www.w3.org/ns/anno.jsonld']),
+      true
+    )
+    assert.strictEqual(_contextid([{ '@vocab': 'http://example.org/terms#' }]), false)
+  })
 })
 
 describe('controllers/utils.js idNegotiation edge cases', () => {
@@ -288,3 +347,131 @@ describe('controllers/utils.js getPagination', () => {
     assert.ok(result.limit < huge, `limit should be clamped below ${huge}`)
   })
 })
+
+describe('utils.js isContainerType', () => {
+  it('detects a container type in a string or a JSON-LD Array of types', () => {
+    assert.strictEqual(utils.isContainerType({ '@type': 'AnnotationPage' }), true)
+    assert.strictEqual(utils.isContainerType({ type: 'sc:AnnotationList' }), true, 'prefixed spellings still match')
+    assert.strictEqual(utils.isContainerType({ type: ['Manifest', 'Collection'] }), true)
+    assert.strictEqual(utils.isContainerType({ type: [null, 42, 'AnnotationPage'] }), true, 'non-string members are skipped, not thrown on')
+    assert.strictEqual(utils.isContainerType({ type: ['Manifest', 'Image'] }), false)
+    assert.strictEqual(utils.isContainerType({}), false)
+  })
+})
+
+describe('controllers/utils.js findLeafAnnotationsFor', () => {
+  const ENTITY_URI = 'https://store.rerum.io/v1/id/entity-id'
+  const SLUG_URI = 'https://store.rerum.io/v1/id/entity-slug'
+  const TARGET_KEYS = ['target', 'target.@id', 'target.id', 'target.source', 'target.source.@id', 'target.source.id']
+
+  let capturedQuery
+  let findCalls
+
+  /**
+   * Point db.find() at a cursor over the given documents and record the filter it was called with.
+   *
+   * @param docs The Annotation documents the cursor will yield.
+   * @return The cursor double, so a test can inspect it.
+   */
+  function armFind(docs = []) {
+    resetMocks()
+    capturedQuery = undefined
+    findCalls = 0
+    const cursor = createCursor(docs)
+    db.find.mockImplementation(query => {
+      findCalls++
+      capturedQuery = query
+      return cursor
+    })
+    return cursor
+  }
+
+  // $and[0] holds the target conditions, $and[1] the Annotation type conditions.
+  const targetConditions = () => capturedQuery.$and[0].$or
+
+  it('constrains the query to the leaf versions, every target key, and every type spelling', async () => {
+    armFind()
+
+    await findLeafAnnotationsFor([ENTITY_URI, SLUG_URI, ENTITY_URI, undefined, ''])
+
+    assert.deepStrictEqual(capturedQuery['__rerum.history.next'], { $exists: true, $size: 0 })
+    for (const targetKey of TARGET_KEYS) {
+      const values = targetConditions()
+        .filter(condition => Object.hasOwn(condition, targetKey))
+        .map(condition => condition[targetKey])
+      assert.strictEqual(values.length, 8, `${targetKey}: two URIs, each in two schemes and as a fragment`)
+      assert.ok(values.includes(ENTITY_URI) && values.includes(ENTITY_URI.replace(/^https/, 'http')))
+      assert.ok(values.includes(SLUG_URI), 'every URI the entity answers to is targeted')
+      const patterns = values.filter(value => value instanceof RegExp)
+      assert.ok(patterns.some(pattern => pattern.test(`${ENTITY_URI}#xywh=0,0,100,100`)), 'a fragment of the URI is a match')
+      assert.ok(
+        patterns.every(pattern => !pattern.test('https://storeXrerum.io/v1/id/entity-id#xywh=0,0,100,100')),
+        'the URI is escaped, so its dots are not wildcards'
+      )
+    }
+    assert.deepStrictEqual(capturedQuery.$and[1].$or, [
+      { type: 'Annotation' },
+      { type: 'oa:Annotation' },
+      { type: 'http://www.w3.org/ns/oa#Annotation' },
+      { type: 'https://www.w3.org/ns/oa#Annotation' },
+      { '@type': 'Annotation' },
+      { '@type': 'oa:Annotation' },
+      { '@type': 'http://www.w3.org/ns/oa#Annotation' },
+      { '@type': 'https://www.w3.org/ns/oa#Annotation' }
+    ])
+
+    armFind()
+    await findLeafAnnotationsFor('bare-slug')
+    assert.strictEqual(targetConditions().length, TARGET_KEYS.length, 'a non-URI target has no scheme or fragment to anticipate')
+  })
+
+  it('ANDs in the supplied filters, doubling the URI scheme for a generator or creator', async () => {
+    armFind()
+
+    await findLeafAnnotationsFor(ENTITY_URI, {
+      '__rerum.generatedBy': 'https://store.rerum.io/v1/id/agent007',
+      creator: 'Fred',
+      motivation: 'describing'
+    })
+
+    assert.deepStrictEqual(capturedQuery.$and.slice(2), [
+      {
+        $or: [
+          { '__rerum.generatedBy': 'http://store.rerum.io/v1/id/agent007' },
+          { '__rerum.generatedBy': 'https://store.rerum.io/v1/id/agent007' }
+        ]
+      },
+      { creator: 'Fred' },
+      { motivation: 'describing' }
+    ])
+  })
+
+  it('returns the matches sorted by _id, with _id dropped, read in batched strides', async () => {
+    const cursor = armFind([
+      { _id: 'ccc', order: 'third' },
+      { _id: 'aaa', order: 'first' },
+      { _id: 'bbb', order: 'second' }
+    ])
+    let requestedBatchSize
+    const chainable = cursor.batchSize
+    cursor.batchSize = size => {
+      requestedBatchSize = size
+      return chainable.call(cursor, size)
+    }
+
+    const result = await findLeafAnnotationsFor(ENTITY_URI)
+
+    assert.deepStrictEqual(result.map(match => match.order), ['first', 'second', 'third'])
+    assert.deepStrictEqual(result.map(match => Object.hasOwn(match, '_id')), [false, false, false])
+    assert.ok(requestedBatchSize > 0, 'the driver must be told how big a stride to transfer')
+  })
+
+  it('returns an empty Array without querying when there is no target', async () => {
+    armFind([{ _id: 'anno1' }])
+
+    const result = await findLeafAnnotationsFor([null, '', undefined])
+
+    assert.deepStrictEqual(result, [])
+    assert.strictEqual(findCalls, 0, 'an empty $or is a MongoDB error, not an empty result')
+  })
+})
diff --git a/routes/__tests__/id.test.js b/routes/__tests__/id.test.js
index e6701fc6..522d64ba 100644
--- a/routes/__tests__/id.test.js
+++ b/routes/__tests__/id.test.js
@@ -9,6 +9,10 @@ import controller from '../../db-controller.js'
 const routeTester = new express()
 routeTester.use(express.json({ type: ["application/json", "application/ld+json"] }))
 
+// The /expanded sub-path is mounted first because the prefix-matching /id/:_id mount below would
+// otherwise swallow it.  This mirrors the route order in routes/id.js.
+routeTester.use("/id/:_id/expanded", controller.idExpanded)
+
 // Mount our own /id route without auth, matching routes/id.js: GET only, no HEAD handler.
 routeTester.use("/id/:_id", controller.id)
 
@@ -31,7 +35,7 @@ const mockDoc = {
 }
 
 // Import db mock so we can configure per-test behaviour
-import { db, resetMocks } from '../../database/index.js'
+import { db, resetMocks, createCursor } from '../../database/index.js'
 
 beforeEach(() => {
   resetMocks()
@@ -121,3 +125,293 @@ describe('id route overwrite headers', () => {
     assert.strictEqual(response.headers['current-overwritten-version'], '')
   })
 })
+
+// Fixtures for GET|POST /id/:_id/expanded.  This record's '@context' is not one of the known
+// id-negotiation contexts, so it keeps its '@id' through the response.
+const EXPAND_ID = "expandme123"
+const EXPAND_URI = `${MOCK_PREFIX}${EXPAND_ID}`
+const EVIL_URI = "https://evil.example.org/hijacked"
+
+const expandableDoc = {
+  _id: EXPAND_ID,
+  "@id": EXPAND_URI,
+  "@context": "http://www.loc.gov/mods",
+  "@type": "named-gloss",
+  title: "A Gloss",
+  __rerum: {
+    generatedBy: MOCK_AGENT,
+    history: { prime: "root", previous: "", next: [] },
+    isReleased: "",
+    isOverwritten: "",
+    releases: { previous: "", next: [], replaces: "" },
+    createdAt: "2025-01-01T00:00:00.000"
+  }
+}
+
+let annoCount = 0
+
+/**
+ * Build a leaf Annotation targeting the expandable record.  Each one needs its own '_id' because
+ * findLeafAnnotationsFor() sorts on it before dropping it, which fixes the merge order.
+ *
+ * @param props The Annotation properties under test, usually a body.
+ * @return An Annotation document.
+ */
+function anno(props) {
+  annoCount++
+  return {
+    _id: `anno${String(annoCount).padStart(3, "0")}`,
+    type: "Annotation",
+    target: EXPAND_URI,
+    ...props
+  }
+}
+
+// The MongoDB filter the controller built for the last expansion, or undefined when it never queried.
+let capturedQuery
+
+/**
+ * Arm the database double for a single /expanded request.
+ *
+ * @param record The document db.findOne() will answer with.
+ * @param annos The Annotation documents the cursor will yield.
+ */
+function armExpansion(record, annos = []) {
+  capturedQuery = undefined
+  db.findOne.mockResolvedValueOnce(structuredClone(record))
+  db.find.mockImplementationOnce(query => {
+    capturedQuery = query
+    return createCursor(annos)
+  })
+}
+
+describe('GET /id/:id/expanded', () => {
+  it("'/id/:id/expanded' route functions", async () => {
+    armExpansion(expandableDoc, [anno({ body: { subject: "history" } })])
+
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(response.statusCode, 200)
+    assert.strictEqual(response.body["@id"], EXPAND_URI)
+    assert.strictEqual(response.body.subject, "history")
+    assert.strictEqual(response.body._id, undefined)
+    assert.strictEqual(Object.keys(response.body).at(-1), '__rerum', '__rerum stays last')
+    assert.strictEqual(response.headers['annotations-gathered'], '1')
+    assert.strictEqual(response.headers['annotations-merged'], '1')
+    assert.strictEqual(response.headers['cache-control'], 'max-age=86400, must-revalidate')
+    assert.strictEqual(response.headers['current-overwritten-version'], '')
+  })
+
+  it('merges the anticipated body formats, collecting collisions into an Array', async () => {
+    const textualBody = { type: "TextualBody", value: "bare spelling", format: "text/plain", language: "en" }
+    const prefixedBody = { "@type": "oa:TextualBody", value: "oa spelling" }
+    const arrayTypedBody = { type: ["TextualBody"], value: "Array-typed spelling" }
+    armExpansion(expandableDoc, [
+      anno({ body: { subject: "history" } }),
+      anno({ body: [{ era: "medieval" }] }),
+      anno({ bodyValue: "the W3C shorthand" }),
+      anno({ body: textualBody }),
+      anno({ body: prefixedBody }),
+      anno({ body: [arrayTypedBody] }),
+      anno({ body: { title: "An Annotated Title" } }),
+      anno({ body: { colors: ["red", "blue"] } }),
+      anno({ body: { colors: ["black"] } })
+    ])
+
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(response.body.subject, "history", 'a single-key body merges as a raw value')
+    assert.strictEqual(response.body.era, "medieval", 'a one-element Array is the same body unwrapped')
+    // A TextualBody is kept whole so its format and language survive, whatever the type spelling.
+    assert.deepStrictEqual(response.body.bodyValue,
+      ["the W3C shorthand", textualBody, prefixedBody, arrayTypedBody])
+    assert.deepStrictEqual(response.body.title, ["A Gloss", "An Annotated Title"], 'the record value comes first')
+    assert.deepStrictEqual(response.body.colors, ["red", "blue", "black"], 'Array contributions flatten')
+  })
+
+  it('does not merge Annotations that make no single assertion', async () => {
+    armExpansion(expandableDoc, [
+      anno({ body: [{ first: "one" }, { second: "two" }] }),
+      anno({ body: { subject: "history", note: "extra" } }),
+      anno({ body: "https://store.rerum.io/v1/id/an-external-body" }),
+      anno({ motivation: "bookmarking" }),
+      anno({ body: { merged: "yes" } })
+    ])
+
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    for (const key of ['first', 'second', 'subject', 'note', 'motivation']) {
+      assert.strictEqual(response.body[key], undefined, `${key} must not be merged`)
+    }
+    assert.strictEqual(response.body.merged, "yes")
+    assert.strictEqual(response.headers['annotations-gathered'], '5')
+    assert.strictEqual(response.headers['annotations-merged'], '1', 'only the contributing Annotation counts')
+  })
+
+  it('never lets an Annotation body overwrite identity or system properties', async () => {
+    armExpansion(expandableDoc, [
+      anno({ body: { "@id": EVIL_URI } }),
+      anno({ body: { id: EVIL_URI } }),
+      anno({ body: { _id: "evil-id" } }),
+      anno({ body: { __rerum: { evil: true } } }),
+      anno({ body: { __deleted: { time: "2025-01-01T00:00:00.000" } } }),
+      anno({ body: { "@context": "https://evil.example.org/context.json" } }),
+      // An object literal with a __proto__ key sets the prototype instead of creating an own
+      // property.  JSON.parse creates the own property, which is what a MongoDB document has.
+      anno({ body: JSON.parse('{"__proto__":{"polluted":"yes"}}') })
+    ])
+
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(response.body["@id"], EXPAND_URI)
+    assert.strictEqual(response.body.id, undefined)
+    assert.strictEqual(response.body._id, undefined)
+    assert.strictEqual(response.body.__rerum.evil, undefined)
+    assert.strictEqual(response.body.__rerum.generatedBy, MOCK_AGENT)
+    assert.strictEqual(response.body.__deleted, undefined)
+    assert.strictEqual(response.body["@context"], "http://www.loc.gov/mods")
+    assert.strictEqual(Object.hasOwn(response.body, '__proto__'), false)
+    assert.strictEqual({}.polluted, undefined, 'Object.prototype must not be polluted')
+    assert.strictEqual(response.headers['annotations-merged'], '0')
+  })
+
+  it('negotiates the id form from the record @context before merging', async () => {
+    const negotiable = structuredClone(expandableDoc)
+    negotiable["@context"] = "http://iiif.io/api/presentation/3/context.json"
+    armExpansion(negotiable, [anno({ body: { subject: "history" } })])
+
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    const negotiatedURI = `${process.env.RERUM_ID_PREFIX}${EXPAND_ID}`
+    assert.strictEqual(response.body.id, negotiatedURI)
+    assert.strictEqual(response.body["@id"], undefined)
+    assert.strictEqual(response.headers.location, negotiatedURI)
+    assert.strictEqual(response.body.subject, "history", 'the expansion still happens')
+  })
+
+  it('gathers Annotations that target the record by its Slug', async () => {
+    const slugged = structuredClone(expandableDoc)
+    slugged.__rerum.slug = "my-slug"
+    armExpansion(slugged, [])
+
+    await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    const exactTargets = capturedQuery.$and[0].$or
+      .filter(condition => typeof condition.target === "string")
+      .map(condition => condition.target)
+    assert.ok(exactTargets.includes(EXPAND_URI), 'the _id URI must be targeted')
+    assert.ok(exactTargets.includes(`${MOCK_PREFIX}my-slug`), 'the slug URI must be targeted too')
+  })
+
+  it('returns the tombstone unexpanded for a deleted record', async () => {
+    const deletedDoc = structuredClone(expandableDoc)
+    deletedDoc.__deleted = { time: "2025-01-01T00:00:00.000", deletor: MOCK_AGENT }
+    armExpansion(deletedDoc, [anno({ body: { subject: "history" } })])
+
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(response.statusCode, 200)
+    assert.ok(response.body.__deleted)
+    assert.strictEqual(response.body.subject, undefined)
+    assert.strictEqual(response.headers['annotations-gathered'], '0')
+    assert.strictEqual(capturedQuery, undefined, 'a deleted record is not queried for Annotations')
+  })
+
+  it('returns 404 when the object is not in RERUM', async () => {
+    db.findOne.mockResolvedValueOnce(null)
+
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(response.statusCode, 404)
+  })
+
+  it('filters the expansion by the generator and creator parameters', async () => {
+    armExpansion(expandableDoc, [])
+
+    await request(routeTester).get(`/id/${EXPAND_ID}/expanded?generator=${MOCK_AGENT}&creator=Fred`)
+
+    // $and[0] is the target condition and $and[1] the Annotation type condition.
+    assert.deepStrictEqual(capturedQuery.$and.slice(2), [
+      {
+        $or: [
+          { "__rerum.generatedBy": MOCK_AGENT.replace(/^https/, "http") },
+          { "__rerum.generatedBy": MOCK_AGENT }
+        ]
+      },
+      { creator: "Fred" }
+    ])
+
+    armExpansion(expandableDoc, [])
+    await request(routeTester).get(`/id/${EXPAND_ID}/expanded?generator=one&generator=two&creator=&limit=5`)
+    assert.deepStrictEqual(capturedQuery.$and.slice(2), [],
+      'repeated, empty, and unrelated parameters are not filters')
+  })
+})
+
+describe('POST /id/:id/expanded', () => {
+  it('reads literal filter keys from the body and ignores the ones the endpoint owns', async () => {
+    armExpansion(expandableDoc, [])
+
+    const response = await request(routeTester)
+      .post(`/id/${EXPAND_ID}/expanded?generator=${MOCK_AGENT}`)
+      .set('Content-Type', 'application/json')
+      .send({
+        target: EVIL_URI,
+        "target.id": EVIL_URI,
+        type: "Dataset",
+        "@type": "Dataset",
+        "__rerum.history": { next: [] },
+        "__rerum.history.next": { $size: 3 },
+        motivation: "describing"
+      })
+
+    assert.strictEqual(response.statusCode, 200)
+    assert.deepStrictEqual(capturedQuery.$and.slice(2), [{ motivation: "describing" }],
+      'the reserved keys and the URL parameters are not filters')
+    assert.strictEqual(response.headers['cache-control'], undefined,
+      'a filtered read is not browser-cacheable')
+  })
+
+  it('expands unfiltered when no body is supplied', async () => {
+    armExpansion(expandableDoc, [anno({ body: { subject: "history" } })])
+
+    const response = await request(routeTester).post(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(response.statusCode, 200)
+    assert.strictEqual(response.body.subject, "history")
+    assert.deepStrictEqual(capturedQuery.$and.slice(2), [])
+  })
+
+  it('returns 400 when the body is an Array instead of a filter object', async () => {
+    db.findOne.mockResolvedValueOnce(structuredClone(expandableDoc))
+
+    const response = await request(routeTester)
+      .post(`/id/${EXPAND_ID}/expanded`)
+      .set('Content-Type', 'application/json')
+      .send([{ motivation: "describing" }])
+
+    assert.strictEqual(response.statusCode, 400)
+  })
+})
+
+// RFC 9110 s9.3.2: HEAD sends the same headers a GET would.
+describe('HEAD /id/:id/expanded', () => {
+  it('sends the same headers and Content-Length as the GET, with no body', async () => {
+    armExpansion(expandableDoc, [anno({ body: { subject: "history" } })])
+    const getResp = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    armExpansion(expandableDoc, [anno({ body: { subject: "history" } })])
+    const headResp = await request(routeTester).head(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(headResp.statusCode, 200)
+    assert.ok(Number(getResp.headers['content-length']) > 0, 'GET must report a Content-Length')
+    assert.strictEqual(headResp.headers['content-length'], getResp.headers['content-length'])
+    assert.ok(headResp.headers['etag'], 'HEAD must report an ETag to validate against')
+    for (const header of ['cache-control', 'etag', 'content-type', 'link', 'allow',
+      'current-overwritten-version', 'location', 'annotations-gathered', 'annotations-merged']) {
+      assert.strictEqual(headResp.headers[header], getResp.headers[header],
+        `HEAD and GET must agree on ${header}`)
+    }
+    assert.ok(!headResp.body || Object.keys(headResp.body).length === 0)
+  })
+})
diff --git a/routes/__tests__/route_wrappers.test.js b/routes/__tests__/route_wrappers.test.js
index e4522a83..47cd4c0f 100644
--- a/routes/__tests__/route_wrappers.test.js
+++ b/routes/__tests__/route_wrappers.test.js
@@ -22,6 +22,7 @@ import searchRouter from '../search.js'
 import queryRouter from '../query.js'
 import releaseRouter from '../release.js'
 import apiRoutesRouter from '../api-routes.js'
+import rest from '../../rest.js'
 import gogFragmentsRouter from '../_gog_fragments_from_manuscript.js'
 import gogGlossesRouter from '../_gog_glosses_from_manuscript.js'
 
@@ -208,6 +209,7 @@ describe('unsupported-method 405 fallbacks', () => {
     { label: '/delete/:_id',    router: deleteRouter,     path: '/:_id' },
     { label: '/history/:_id',   router: historyRouter,    path: '/:_id' },
     { label: '/id/:_id',        router: idRouter,         path: '/:_id' },
+    { label: '/id/:_id/expanded', router: idRouter,       path: '/:_id/expanded' },
     { label: '/since/:_id',     router: sinceRouter,      path: '/:_id' },
     { label: '/_gog_fragments_from_manuscript', router: gogFragmentsRouter, path: '/' },
     { label: '/_gog_glosses_from_manuscript',   router: gogGlossesRouter,   path: '/' }
@@ -220,6 +222,15 @@ describe('unsupported-method 405 fallbacks', () => {
   }
 })
 
+describe('id route content-type wiring', () => {
+  it('runs the JSON content-type check before the expanded controller on POST', () => {
+    const postLayers = getMethodLayers(idRouter, '/:_id/expanded', 'post')
+
+    assert.strictEqual(postLayers.length, 2, 'Expected a middleware layer and a controller layer')
+    assert.strictEqual(postLayers[0].handle, rest.verifyJsonContentType)
+  })
+})
+
 describe('api routes discovery', () => {
   it('directly serves the welcome page from the static route handler', () => {
     const handler = getMethodLayers(staticRouter, '/', 'get').at(-1)

From 6a039b3a06634423f7b8c4d1f577f3abb36c8e20 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 12:49:52 -0500
Subject: [PATCH 43/48] Reduce and simplify

---
 __tests__/routes_mounted.test.js        |  2 -
 __tests__/utils.test.js                 | 84 +++----------------------
 routes/__tests__/id.test.js             | 46 ++++++--------
 routes/__tests__/route_wrappers.test.js | 10 ---
 4 files changed, 27 insertions(+), 115 deletions(-)

diff --git a/__tests__/routes_mounted.test.js b/__tests__/routes_mounted.test.js
index 3e40f9aa..abc16814 100644
--- a/__tests__/routes_mounted.test.js
+++ b/__tests__/routes_mounted.test.js
@@ -27,8 +27,6 @@ const mountedApiRoutes = [
   { name: '/v1/api/release/{id}', method: 'patch', path: '/v1/api/release/test-mounted-id' },
   { name: '/v1/api/search', method: 'post', path: '/v1/api/search', headers: { 'Content-Type': 'text/plain' }, body: 'mounted search' },
   { name: '/v1/api/search/phrase', method: 'post', path: '/v1/api/search/phrase', headers: { 'Content-Type': 'text/plain' }, body: 'mounted phrase search' },
-  // A missing record answers 404 here, which an unmounted path would too.  PUT is the
-  // method this route rejects, so a non-404 proves the router is wired up.
   { name: '/v1/id/{_id}/expanded', method: 'put', path: '/v1/id/test-mounted-id/expanded' }
 ]
 
diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js
index 3cb20811..d57fa28c 100644
--- a/__tests__/utils.test.js
+++ b/__tests__/utils.test.js
@@ -72,7 +72,7 @@ describe('utils.js configureRerumOptions', () => {
     assert.strictEqual(imported.__rerum.releases.previous, '')
   })
 
-  it('carries the version and release chain forward when updating', () => {
+  it('carries the version and release chain forward when updating an existing object', () => {
     const fromRoot = utils.configureRerumOptions(
       AGENT,
       { '@id': RECEIVED_ID, __rerum: { history: { prime: 'root', previous: '', next: [] } } },
@@ -263,6 +263,12 @@ describe('controllers/utils.js _contextid', () => {
       _contextid(['http://example.com/other', 'http://iiif.io/api/presentation/3/context.json']),
       true
     )
+    // An inline term definition object, or any other non-string member, names no context.
+    assert.strictEqual(
+      _contextid([{ '@vocab': 'http://example.org/terms#' }, 'http://www.w3.org/ns/anno.jsonld']),
+      true
+    )
+    assert.strictEqual(_contextid([{ '@vocab': 'http://example.org/terms#' }]), false)
   })
 
   it('returns false for non-string, non-array input', () => {
@@ -270,14 +276,6 @@ describe('controllers/utils.js _contextid', () => {
     assert.strictEqual(_contextid(123), false)
     assert.strictEqual(_contextid({}), false)
   })
-
-  it('skips non-string members of an array, such as an inline term definition', () => {
-    assert.strictEqual(
-      _contextid([{ '@vocab': 'http://example.org/terms#' }, 'http://www.w3.org/ns/anno.jsonld']),
-      true
-    )
-    assert.strictEqual(_contextid([{ '@vocab': 'http://example.org/terms#' }]), false)
-  })
 })
 
 describe('controllers/utils.js idNegotiation edge cases', () => {
@@ -348,42 +346,25 @@ describe('controllers/utils.js getPagination', () => {
   })
 })
 
-describe('utils.js isContainerType', () => {
-  it('detects a container type in a string or a JSON-LD Array of types', () => {
-    assert.strictEqual(utils.isContainerType({ '@type': 'AnnotationPage' }), true)
-    assert.strictEqual(utils.isContainerType({ type: 'sc:AnnotationList' }), true, 'prefixed spellings still match')
-    assert.strictEqual(utils.isContainerType({ type: ['Manifest', 'Collection'] }), true)
-    assert.strictEqual(utils.isContainerType({ type: [null, 42, 'AnnotationPage'] }), true, 'non-string members are skipped, not thrown on')
-    assert.strictEqual(utils.isContainerType({ type: ['Manifest', 'Image'] }), false)
-    assert.strictEqual(utils.isContainerType({}), false)
-  })
-})
-
 describe('controllers/utils.js findLeafAnnotationsFor', () => {
   const ENTITY_URI = 'https://store.rerum.io/v1/id/entity-id'
   const SLUG_URI = 'https://store.rerum.io/v1/id/entity-slug'
   const TARGET_KEYS = ['target', 'target.@id', 'target.id', 'target.source', 'target.source.@id', 'target.source.id']
 
   let capturedQuery
-  let findCalls
 
   /**
    * Point db.find() at a cursor over the given documents and record the filter it was called with.
    *
    * @param docs The Annotation documents the cursor will yield.
-   * @return The cursor double, so a test can inspect it.
    */
   function armFind(docs = []) {
     resetMocks()
     capturedQuery = undefined
-    findCalls = 0
-    const cursor = createCursor(docs)
     db.find.mockImplementation(query => {
-      findCalls++
       capturedQuery = query
-      return cursor
+      return createCursor(docs)
     })
-    return cursor
   }
 
   // $and[0] holds the target conditions, $and[1] the Annotation type conditions.
@@ -425,53 +406,4 @@ describe('controllers/utils.js findLeafAnnotationsFor', () => {
     assert.strictEqual(targetConditions().length, TARGET_KEYS.length, 'a non-URI target has no scheme or fragment to anticipate')
   })
 
-  it('ANDs in the supplied filters, doubling the URI scheme for a generator or creator', async () => {
-    armFind()
-
-    await findLeafAnnotationsFor(ENTITY_URI, {
-      '__rerum.generatedBy': 'https://store.rerum.io/v1/id/agent007',
-      creator: 'Fred',
-      motivation: 'describing'
-    })
-
-    assert.deepStrictEqual(capturedQuery.$and.slice(2), [
-      {
-        $or: [
-          { '__rerum.generatedBy': 'http://store.rerum.io/v1/id/agent007' },
-          { '__rerum.generatedBy': 'https://store.rerum.io/v1/id/agent007' }
-        ]
-      },
-      { creator: 'Fred' },
-      { motivation: 'describing' }
-    ])
-  })
-
-  it('returns the matches sorted by _id, with _id dropped, read in batched strides', async () => {
-    const cursor = armFind([
-      { _id: 'ccc', order: 'third' },
-      { _id: 'aaa', order: 'first' },
-      { _id: 'bbb', order: 'second' }
-    ])
-    let requestedBatchSize
-    const chainable = cursor.batchSize
-    cursor.batchSize = size => {
-      requestedBatchSize = size
-      return chainable.call(cursor, size)
-    }
-
-    const result = await findLeafAnnotationsFor(ENTITY_URI)
-
-    assert.deepStrictEqual(result.map(match => match.order), ['first', 'second', 'third'])
-    assert.deepStrictEqual(result.map(match => Object.hasOwn(match, '_id')), [false, false, false])
-    assert.ok(requestedBatchSize > 0, 'the driver must be told how big a stride to transfer')
-  })
-
-  it('returns an empty Array without querying when there is no target', async () => {
-    armFind([{ _id: 'anno1' }])
-
-    const result = await findLeafAnnotationsFor([null, '', undefined])
-
-    assert.deepStrictEqual(result, [])
-    assert.strictEqual(findCalls, 0, 'an empty $or is a MongoDB error, not an empty result')
-  })
 })
diff --git a/routes/__tests__/id.test.js b/routes/__tests__/id.test.js
index 522d64ba..77c49753 100644
--- a/routes/__tests__/id.test.js
+++ b/routes/__tests__/id.test.js
@@ -126,8 +126,7 @@ describe('id route overwrite headers', () => {
   })
 })
 
-// Fixtures for GET|POST /id/:_id/expanded.  This record's '@context' is not one of the known
-// id-negotiation contexts, so it keeps its '@id' through the response.
+// Fixtures for GET|POST /id/:_id/expanded.
 const EXPAND_ID = "expandme123"
 const EXPAND_URI = `${MOCK_PREFIX}${EXPAND_ID}`
 const EVIL_URI = "https://evil.example.org/hijacked"
@@ -200,13 +199,25 @@ describe('GET /id/:id/expanded', () => {
     assert.strictEqual(response.headers['annotations-merged'], '1')
     assert.strictEqual(response.headers['cache-control'], 'max-age=86400, must-revalidate')
     assert.strictEqual(response.headers['current-overwritten-version'], '')
+
+    // A container-typed record gets the Web Annotation Link header, including when its type is
+    // serialized as a JSON-LD Array.
+    const container = structuredClone(expandableDoc)
+    container["@type"] = ["Manifest", "AnnotationPage"]
+    armExpansion(container, [])
+    const containerResp = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+    assert.match(containerResp.headers.link, /anno\.jsonld/)
+
+    db.findOne.mockResolvedValueOnce(null)
+    const missResp = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+    assert.strictEqual(missResp.statusCode, 404, 'a record that is not in RERUM has no expansion')
   })
 
   it('merges the anticipated body formats, collecting collisions into an Array', async () => {
     const textualBody = { type: "TextualBody", value: "bare spelling", format: "text/plain", language: "en" }
     const prefixedBody = { "@type": "oa:TextualBody", value: "oa spelling" }
     const arrayTypedBody = { type: ["TextualBody"], value: "Array-typed spelling" }
-    armExpansion(expandableDoc, [
+    const gathered = [
       anno({ body: { subject: "history" } }),
       anno({ body: [{ era: "medieval" }] }),
       anno({ bodyValue: "the W3C shorthand" }),
@@ -216,7 +227,10 @@ describe('GET /id/:id/expanded', () => {
       anno({ body: { title: "An Annotated Title" } }),
       anno({ body: { colors: ["red", "blue"] } }),
       anno({ body: { colors: ["black"] } })
-    ])
+    ]
+    // The query plan promises no order, so hand them over reversed.  The expansion sorts by '_id'
+    // before merging, which is what makes the assembled entity reproducible.
+    armExpansion(expandableDoc, [...gathered].reverse())
 
     const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
 
@@ -275,20 +289,6 @@ describe('GET /id/:id/expanded', () => {
     assert.strictEqual(response.headers['annotations-merged'], '0')
   })
 
-  it('negotiates the id form from the record @context before merging', async () => {
-    const negotiable = structuredClone(expandableDoc)
-    negotiable["@context"] = "http://iiif.io/api/presentation/3/context.json"
-    armExpansion(negotiable, [anno({ body: { subject: "history" } })])
-
-    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
-
-    const negotiatedURI = `${process.env.RERUM_ID_PREFIX}${EXPAND_ID}`
-    assert.strictEqual(response.body.id, negotiatedURI)
-    assert.strictEqual(response.body["@id"], undefined)
-    assert.strictEqual(response.headers.location, negotiatedURI)
-    assert.strictEqual(response.body.subject, "history", 'the expansion still happens')
-  })
-
   it('gathers Annotations that target the record by its Slug', async () => {
     const slugged = structuredClone(expandableDoc)
     slugged.__rerum.slug = "my-slug"
@@ -317,14 +317,6 @@ describe('GET /id/:id/expanded', () => {
     assert.strictEqual(capturedQuery, undefined, 'a deleted record is not queried for Annotations')
   })
 
-  it('returns 404 when the object is not in RERUM', async () => {
-    db.findOne.mockResolvedValueOnce(null)
-
-    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
-
-    assert.strictEqual(response.statusCode, 404)
-  })
-
   it('filters the expansion by the generator and creator parameters', async () => {
     armExpansion(expandableDoc, [])
 
@@ -382,7 +374,7 @@ describe('POST /id/:id/expanded', () => {
     assert.deepStrictEqual(capturedQuery.$and.slice(2), [])
   })
 
-  it('returns 400 when the body is an Array instead of a filter object', async () => {
+  it('returns 400 when the body is not a filter object', async () => {
     db.findOne.mockResolvedValueOnce(structuredClone(expandableDoc))
 
     const response = await request(routeTester)
diff --git a/routes/__tests__/route_wrappers.test.js b/routes/__tests__/route_wrappers.test.js
index 47cd4c0f..c0bb1300 100644
--- a/routes/__tests__/route_wrappers.test.js
+++ b/routes/__tests__/route_wrappers.test.js
@@ -22,7 +22,6 @@ import searchRouter from '../search.js'
 import queryRouter from '../query.js'
 import releaseRouter from '../release.js'
 import apiRoutesRouter from '../api-routes.js'
-import rest from '../../rest.js'
 import gogFragmentsRouter from '../_gog_fragments_from_manuscript.js'
 import gogGlossesRouter from '../_gog_glosses_from_manuscript.js'
 
@@ -222,15 +221,6 @@ describe('unsupported-method 405 fallbacks', () => {
   }
 })
 
-describe('id route content-type wiring', () => {
-  it('runs the JSON content-type check before the expanded controller on POST', () => {
-    const postLayers = getMethodLayers(idRouter, '/:_id/expanded', 'post')
-
-    assert.strictEqual(postLayers.length, 2, 'Expected a middleware layer and a controller layer')
-    assert.strictEqual(postLayers[0].handle, rest.verifyJsonContentType)
-  })
-})
-
 describe('api routes discovery', () => {
   it('directly serves the welcome page from the static route handler', () => {
     const handler = getMethodLayers(staticRouter, '/', 'get').at(-1)

From a76195a35fcf6cb69e83ecaeded598c4441134c3 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 12:52:57 -0500
Subject: [PATCH 44/48] Reduce and simplify

---
 routes/__tests__/id.test.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/routes/__tests__/id.test.js b/routes/__tests__/id.test.js
index 77c49753..45e4ee71 100644
--- a/routes/__tests__/id.test.js
+++ b/routes/__tests__/id.test.js
@@ -9,8 +9,6 @@ import controller from '../../db-controller.js'
 const routeTester = new express()
 routeTester.use(express.json({ type: ["application/json", "application/ld+json"] }))
 
-// The /expanded sub-path is mounted first because the prefix-matching /id/:_id mount below would
-// otherwise swallow it.  This mirrors the route order in routes/id.js.
 routeTester.use("/id/:_id/expanded", controller.idExpanded)
 
 // Mount our own /id route without auth, matching routes/id.js: GET only, no HEAD handler.

From 2c1113b263098c2388015b12bdd3cae16761e5cb Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 13:59:39 -0500
Subject: [PATCH 45/48] changes during review

---
 __tests__/core_provider_contract.test.js |  2 +-
 __tests__/utils.test.js                  | 17 ++++++++-
 routes/__tests__/id.test.js              | 44 +++++++++++++++++++-----
 3 files changed, 52 insertions(+), 11 deletions(-)

diff --git a/__tests__/core_provider_contract.test.js b/__tests__/core_provider_contract.test.js
index 6b34a726..32b0eb9a 100644
--- a/__tests__/core_provider_contract.test.js
+++ b/__tests__/core_provider_contract.test.js
@@ -202,13 +202,13 @@ const requiredResponseCodes = {
   // The expanded reads are covered in routes/__tests__/id.test.js: 404 on a miss, and a
   // POST that reads filters from its body, so it also answers the body-related codes.
   'GET /id/{id}/expanded': ['200', '404'],
-  'HEAD /id/{id}/expanded': ['200', '404'],
   'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'],
   'GET /since/{id}': ['200', '404'],
   'GET /history/{id}': ['200', '404'],
   // HEAD parity tests in routes/__tests__/{id,since,history,query}.test.js assert 404 on miss;
   // enforce that the contract declares the same so drift on either side is caught.
   'HEAD /id/{id}': ['200', '404'],
+  'HEAD /id/{id}/expanded': ['200', '404'],
   'HEAD /since/{id}': ['200', '404'],
   'HEAD /history/{id}': ['200', '404'],
   'HEAD /api/query': ['200', '404'],
diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js
index d57fa28c..8ec29a8f 100644
--- a/__tests__/utils.test.js
+++ b/__tests__/utils.test.js
@@ -54,6 +54,9 @@ describe('utils.js configureRerumOptions', () => {
   const AGENT = 'https://store.rerum.io/v1/id/legitimate-agent'
   const RECEIVED_ID = 'https://store.rerum.io/v1/id/received-id'
   const FORGED = {
+    generatedBy: 'https://attacker.example/forged',
+    isReleased: '2020-01-01T00:00:00.000',
+    isOverwritten: '2020-01-01T00:00:00.000',
     history: { prime: 'https://store.rerum.io/v1/id/forged-prime', previous: 'https://store.rerum.io/v1/id/forged-previous', next: ['https://store.rerum.io/v1/id/forged-next'] },
     releases: { previous: 'https://store.rerum.io/v1/id/forged-release', next: [], replaces: '' }
   }
@@ -64,6 +67,10 @@ describe('utils.js configureRerumOptions', () => {
     assert.strictEqual(created.__rerum.history.previous, '')
     assert.deepStrictEqual(created.__rerum.history.next, [])
     assert.strictEqual(created.__rerum.releases.previous, '')
+    // isReleased gates release and overwrite, isOverwritten is the optimistic locking token.
+    assert.strictEqual(created.__rerum.generatedBy, AGENT, 'attribution cannot be forged')
+    assert.strictEqual(created.__rerum.isReleased, '', 'a client cannot mint a pre-released object')
+    assert.strictEqual(created.__rerum.isOverwritten, '', 'a client cannot mint a locking token')
 
     // An external object imported through an update is also a root, but it remembers its external self.
     const imported = utils.configureRerumOptions(AGENT, { '@id': 'https://elsewhere.example.org/thing', __rerum: structuredClone(FORGED) }, false, true)
@@ -361,7 +368,7 @@ describe('controllers/utils.js findLeafAnnotationsFor', () => {
   function armFind(docs = []) {
     resetMocks()
     capturedQuery = undefined
-    db.find.mockImplementation(query => {
+    db.find.mockImplementationOnce(query => {
       capturedQuery = query
       return createCursor(docs)
     })
@@ -406,4 +413,12 @@ describe('controllers/utils.js findLeafAnnotationsFor', () => {
     assert.strictEqual(targetConditions().length, TARGET_KEYS.length, 'a non-URI target has no scheme or fragment to anticipate')
   })
 
+  it('gathers nothing rather than querying on an empty $or when there is no target', async () => {
+    // An empty '$or' is a MongoDB error, not an empty result, so the query is never sent.
+    armFind([{ _id: 'anno001', type: 'Annotation' }])
+
+    assert.deepStrictEqual(await findLeafAnnotationsFor([undefined, '', null]), [])
+    assert.deepStrictEqual(await findLeafAnnotationsFor(undefined), [])
+    assert.strictEqual(capturedQuery, undefined, 'nothing to target is nothing to gather')
+  })
 })
diff --git a/routes/__tests__/id.test.js b/routes/__tests__/id.test.js
index 45e4ee71..378f79ca 100644
--- a/routes/__tests__/id.test.js
+++ b/routes/__tests__/id.test.js
@@ -197,18 +197,25 @@ describe('GET /id/:id/expanded', () => {
     assert.strictEqual(response.headers['annotations-merged'], '1')
     assert.strictEqual(response.headers['cache-control'], 'max-age=86400, must-revalidate')
     assert.strictEqual(response.headers['current-overwritten-version'], '')
+  })
 
-    // A container-typed record gets the Web Annotation Link header, including when its type is
-    // serialized as a JSON-LD Array.
+  it('sends the Web Annotation Link header for a container-typed record', async () => {
+    // The type may be serialized as a JSON-LD Array, which still names a container.
     const container = structuredClone(expandableDoc)
     container["@type"] = ["Manifest", "AnnotationPage"]
     armExpansion(container, [])
-    const containerResp = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
-    assert.match(containerResp.headers.link, /anno\.jsonld/)
 
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    assert.match(response.headers.link, /anno\.jsonld/)
+  })
+
+  it('returns 404 when the object is not in RERUM', async () => {
     db.findOne.mockResolvedValueOnce(null)
-    const missResp = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
-    assert.strictEqual(missResp.statusCode, 404, 'a record that is not in RERUM has no expansion')
+
+    const response = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(response.statusCode, 404, 'a record that is not in RERUM has no expansion')
   })
 
   it('merges the anticipated body formats, collecting collisions into an Array', async () => {
@@ -365,7 +372,11 @@ describe('POST /id/:id/expanded', () => {
   it('expands unfiltered when no body is supplied', async () => {
     armExpansion(expandableDoc, [anno({ body: { subject: "history" } })])
 
-    const response = await request(routeTester).post(`/id/${EXPAND_ID}/expanded`)
+    // routes/id.js gates POST behind rest.verifyJsonContentType, so a client reaches the
+    // unfiltered expansion by declaring JSON and sending nothing, not by omitting Content-Type.
+    const response = await request(routeTester)
+      .post(`/id/${EXPAND_ID}/expanded`)
+      .set('Content-Type', 'application/json')
 
     assert.strictEqual(response.statusCode, 200)
     assert.strictEqual(response.body.subject, "history")
@@ -373,7 +384,8 @@ describe('POST /id/:id/expanded', () => {
   })
 
   it('returns 400 when the body is not a filter object', async () => {
-    db.findOne.mockResolvedValueOnce(structuredClone(expandableDoc))
+    // Nothing is armed.  A rejected body must not reach the record read or the Annotation query.
+    capturedQuery = undefined
 
     const response = await request(routeTester)
       .post(`/id/${EXPAND_ID}/expanded`)
@@ -381,11 +393,20 @@ describe('POST /id/:id/expanded', () => {
       .send([{ motivation: "describing" }])
 
     assert.strictEqual(response.statusCode, 400)
+    assert.strictEqual(capturedQuery, undefined, 'a malformed filter body is rejected before any read')
   })
 })
 
 // RFC 9110 s9.3.2: HEAD sends the same headers a GET would.
 describe('HEAD /id/:id/expanded', () => {
+  it('returns 404 when the object is not in RERUM', async () => {
+    db.findOne.mockResolvedValueOnce(null)
+
+    const response = await request(routeTester).head(`/id/${EXPAND_ID}/expanded`)
+
+    assert.strictEqual(response.statusCode, 404)
+  })
+
   it('sends the same headers and Content-Length as the GET, with no body', async () => {
     armExpansion(expandableDoc, [anno({ body: { subject: "history" } })])
     const getResp = await request(routeTester).get(`/id/${EXPAND_ID}/expanded`)
@@ -397,8 +418,13 @@ describe('HEAD /id/:id/expanded', () => {
     assert.ok(Number(getResp.headers['content-length']) > 0, 'GET must report a Content-Length')
     assert.strictEqual(headResp.headers['content-length'], getResp.headers['content-length'])
     assert.ok(headResp.headers['etag'], 'HEAD must report an ETag to validate against')
+    // Unlike GET /id/:_id, the expansion sends no Last-Modified.  It is assembled from the record
+    // and its Annotations, which have no single modification time, so revalidate on the ETag.
+    assert.strictEqual(getResp.headers['last-modified'], undefined,
+      'an assembled entity has no single modification time to report')
     for (const header of ['cache-control', 'etag', 'content-type', 'link', 'allow',
-      'current-overwritten-version', 'location', 'annotations-gathered', 'annotations-merged']) {
+      'current-overwritten-version', 'location', 'annotations-gathered', 'annotations-merged',
+      'last-modified']) {
       assert.strictEqual(headResp.headers[header], getResp.headers[header],
         `HEAD and GET must agree on ${header}`)
     }

From 8b856c1e1cb8247401dd915ba10ae41fb7e7e8c7 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 14:26:35 -0500
Subject: [PATCH 46/48] changes during review, good to go

---
 __tests__/core_provider_contract.test.js | 5 +++--
 routes/__tests__/id.test.js              | 6 ++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/__tests__/core_provider_contract.test.js b/__tests__/core_provider_contract.test.js
index 32b0eb9a..0ebe3c40 100644
--- a/__tests__/core_provider_contract.test.js
+++ b/__tests__/core_provider_contract.test.js
@@ -199,8 +199,9 @@ const requiredResponseCodes = {
   // 409 is reachable via slug conflict (utils.createExpressError maps code 11000 → 409).
   'PATCH /api/release/{id}': ['200', '400', '401', '403', '404', '409'],
   'GET /id/{id}': ['200', '404'],
-  // The expanded reads are covered in routes/__tests__/id.test.js: 404 on a miss, and a
-  // POST that reads filters from its body, so it also answers the body-related codes.
+  // 200/400/404 are asserted in routes/__tests__/id.test.js.  413 comes from the global 5mb
+  // express.json limit in app.js and 415 from rest.verifyJsonContentType, both covered
+  // generically in routes/__tests__/{routes_mounted,contentType}.test.js as for the /api routes.
   'GET /id/{id}/expanded': ['200', '404'],
   'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'],
   'GET /since/{id}': ['200', '404'],
diff --git a/routes/__tests__/id.test.js b/routes/__tests__/id.test.js
index 378f79ca..5d9f9773 100644
--- a/routes/__tests__/id.test.js
+++ b/routes/__tests__/id.test.js
@@ -37,6 +37,8 @@ import { db, resetMocks, createCursor } from '../../database/index.js'
 
 beforeEach(() => {
   resetMocks()
+  // Restart the anno() sequence below, so each test's merge order is its own call order.
+  annoCount = 0
 })
 
 it("'/id/:id' route functions", async () => {
@@ -325,7 +327,7 @@ describe('GET /id/:id/expanded', () => {
   it('filters the expansion by the generator and creator parameters', async () => {
     armExpansion(expandableDoc, [])
 
-    await request(routeTester).get(`/id/${EXPAND_ID}/expanded?generator=${MOCK_AGENT}&creator=Fred`)
+    await request(routeTester).get(`/id/${EXPAND_ID}/expanded?generator=${encodeURIComponent(MOCK_AGENT)}&creator=Fred`)
 
     // $and[0] is the target condition and $and[1] the Annotation type condition.
     assert.deepStrictEqual(capturedQuery.$and.slice(2), [
@@ -350,7 +352,7 @@ describe('POST /id/:id/expanded', () => {
     armExpansion(expandableDoc, [])
 
     const response = await request(routeTester)
-      .post(`/id/${EXPAND_ID}/expanded?generator=${MOCK_AGENT}`)
+      .post(`/id/${EXPAND_ID}/expanded?generator=${encodeURIComponent(MOCK_AGENT)}`)
       .set('Content-Type', 'application/json')
       .send({
         target: EVIL_URI,

From a0d8e44c8b7c3f26bf42b0744bda25988d17950f Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 14:36:28 -0500
Subject: [PATCH 47/48] small comment cleanup.  Ready to start deploying to
 dev.

---
 __tests__/core_provider_contract.test.js | 4 +---
 controllers/utils.js                     | 8 +++-----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/__tests__/core_provider_contract.test.js b/__tests__/core_provider_contract.test.js
index 0ebe3c40..388981b1 100644
--- a/__tests__/core_provider_contract.test.js
+++ b/__tests__/core_provider_contract.test.js
@@ -199,9 +199,7 @@ const requiredResponseCodes = {
   // 409 is reachable via slug conflict (utils.createExpressError maps code 11000 → 409).
   'PATCH /api/release/{id}': ['200', '400', '401', '403', '404', '409'],
   'GET /id/{id}': ['200', '404'],
-  // 200/400/404 are asserted in routes/__tests__/id.test.js.  413 comes from the global 5mb
-  // express.json limit in app.js and 415 from rest.verifyJsonContentType, both covered
-  // generically in routes/__tests__/{routes_mounted,contentType}.test.js as for the /api routes.
+  // 200/400/404 are asserted in routes/__tests__/id.test.js.  413 comes from the global 5mb express.json limit
   'GET /id/{id}/expanded': ['200', '404'],
   'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'],
   'GET /since/{id}': ['200', '404'],
diff --git a/controllers/utils.js b/controllers/utils.js
index d6ebc2fb..a5c09bc5 100644
--- a/controllers/utils.js
+++ b/controllers/utils.js
@@ -152,11 +152,9 @@ function escapeRegex(literal) {
  *
  * Only 'target' and 'body' are read, whatever the type spelling says.
  *
- * Any entity can answer to more than one URI.  A record minted with a Slug resolves at both
- * '/id/<_id>' and '/id/', and an Annotation may legitimately target it by
- * either one.  Every match is gathered, in a single cursor the driver pages in strides of
- * EXPANSION_BATCH_SIZE.  The query plan yields no order, so the result is sorted by '_id' before it
- * is returned.  That is roughly Annotation creation order, and it makes the expansion reproducible.
+ * Any entity can answer to more than one URI because of Slugs  Every match is gathered, in a single cursor.
+ * The result is sorted by '_id' before it is returned.  That is roughly Annotation creation order, and it makes
+ * the expansion reproducible.
  *
  * @param targetIds The '@id' or 'id' URI of the entity being expanded, or an Array of the URIs it
  * is known by when it answers to more than one.

From b8eed21ec307af1c74d43984ded2138c6e212031 Mon Sep 17 00:00:00 2001
From: Bryan Haberberger 
Date: Thu, 20 Aug 2026 17:41:14 -0500
Subject: [PATCH 48/48] a little cleanup

---
 __tests__/core_provider_contract.test.js | 2 +-
 controllers/crud.js                      | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/__tests__/core_provider_contract.test.js b/__tests__/core_provider_contract.test.js
index 388981b1..7d983a60 100644
--- a/__tests__/core_provider_contract.test.js
+++ b/__tests__/core_provider_contract.test.js
@@ -199,7 +199,7 @@ const requiredResponseCodes = {
   // 409 is reachable via slug conflict (utils.createExpressError maps code 11000 → 409).
   'PATCH /api/release/{id}': ['200', '400', '401', '403', '404', '409'],
   'GET /id/{id}': ['200', '404'],
-  // 200/400/404 are asserted in routes/__tests__/id.test.js.  413 comes from the global 5mb express.json limit
+  // 200/400/404 are asserted in routes/__tests__/id.test.js.  413 and 415 come from Express handlers.
   'GET /id/{id}/expanded': ['200', '404'],
   'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'],
   'GET /since/{id}': ['200', '404'],
diff --git a/controllers/crud.js b/controllers/crud.js
index 05b3343b..7d1f90dc 100644
--- a/controllers/crud.js
+++ b/controllers/crud.js
@@ -277,8 +277,7 @@ const idExpanded = async function (req, res, next) {
         // How many of the Annotations contribute an assertion.  May be less than annotations gathered.
         // The count reports the Annotations that actually alter the entity.
         const merged = annos
-            .map(anno => assertionsFrom(anno)
-            .filter(([key]) => !PROTECTED_EXPANSION_KEYS.has(key)))
+            .map(anno => assertionsFrom(anno).filter(([key]) => !PROTECTED_EXPANSION_KEYS.has(key)))
             .filter(assertions => assertions.length > 0)
         res.set('Annotations-Merged', String(merged.length))
         // Negotiate first, so identity is settled from the record's own '@context' before anything is merged.