diff --git a/__tests__/core_provider_contract.test.js b/__tests__/core_provider_contract.test.js index c73d1309..7d983a60 100644 --- a/__tests__/core_provider_contract.test.js +++ b/__tests__/core_provider_contract.test.js @@ -199,11 +199,15 @@ const requiredResponseCodes = { // 409 is reachable via slug conflict (utils.createExpressError maps code 11000 → 409). 'PATCH /api/release/{id}': ['200', '400', '401', '403', '404', '409'], 'GET /id/{id}': ['200', '404'], + // 200/400/404 are asserted in routes/__tests__/id.test.js. 413 and 415 come from Express handlers. + 'GET /id/{id}/expanded': ['200', '404'], + 'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'], 'GET /since/{id}': ['200', '404'], 'GET /history/{id}': ['200', '404'], // HEAD parity tests in routes/__tests__/{id,since,history,query}.test.js assert 404 on miss; // enforce that the contract declares the same so drift on either side is caught. 'HEAD /id/{id}': ['200', '404'], + 'HEAD /id/{id}/expanded': ['200', '404'], 'HEAD /since/{id}': ['200', '404'], 'HEAD /history/{id}': ['200', '404'], 'HEAD /api/query': ['200', '404'], diff --git a/__tests__/routes_mounted.test.js b/__tests__/routes_mounted.test.js index aacb2094..abc16814 100644 --- a/__tests__/routes_mounted.test.js +++ b/__tests__/routes_mounted.test.js @@ -26,7 +26,8 @@ const mountedApiRoutes = [ { name: '/v1/api/delete/{id}', method: 'delete', path: '/v1/api/delete/test-mounted-id' }, { name: '/v1/api/release/{id}', method: 'patch', path: '/v1/api/release/test-mounted-id' }, { name: '/v1/api/search', method: 'post', path: '/v1/api/search', headers: { 'Content-Type': 'text/plain' }, body: 'mounted search' }, - { name: '/v1/api/search/phrase', method: 'post', path: '/v1/api/search/phrase', headers: { 'Content-Type': 'text/plain' }, body: 'mounted phrase search' } + { name: '/v1/api/search/phrase', method: 'post', path: '/v1/api/search/phrase', headers: { 'Content-Type': 'text/plain' }, body: 'mounted phrase search' }, + { name: '/v1/id/{_id}/expanded', method: 'put', path: '/v1/id/test-mounted-id/expanded' } ] describe('Mounted route surface', () => { diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js index 900ca7c3..8ec29a8f 100644 --- a/__tests__/utils.test.js +++ b/__tests__/utils.test.js @@ -9,9 +9,10 @@ import { parseDocumentID, _contextid, idNegotiation, - getPagination + getPagination, + findLeafAnnotationsFor } from '../controllers/utils.js' -import { db, resetMocks } from '../database/index.js' +import { db, resetMocks, createCursor } from '../database/index.js' describe('utils.js auth gates', () => { it('isDeleted returns true only for objects with __deleted', () => { @@ -49,6 +50,63 @@ describe('utils.js configureRerumOptions', () => { ) assert.strictEqual(result.__rerum.generatedBy, 'https://store.rerum.io/v1/id/legitimate-agent') }) + + const AGENT = 'https://store.rerum.io/v1/id/legitimate-agent' + const RECEIVED_ID = 'https://store.rerum.io/v1/id/received-id' + const FORGED = { + generatedBy: 'https://attacker.example/forged', + isReleased: '2020-01-01T00:00:00.000', + isOverwritten: '2020-01-01T00:00:00.000', + history: { prime: 'https://store.rerum.io/v1/id/forged-prime', previous: 'https://store.rerum.io/v1/id/forged-previous', next: ['https://store.rerum.io/v1/id/forged-next'] }, + releases: { previous: 'https://store.rerum.io/v1/id/forged-release', next: [], replaces: '' } + } + + it('ignores a client-supplied __rerum when minting a new object', () => { + const created = utils.configureRerumOptions(AGENT, { '@id': RECEIVED_ID, __rerum: structuredClone(FORGED) }, false, false) + assert.strictEqual(created.__rerum.history.prime, 'root') + assert.strictEqual(created.__rerum.history.previous, '') + assert.deepStrictEqual(created.__rerum.history.next, []) + assert.strictEqual(created.__rerum.releases.previous, '') + // isReleased gates release and overwrite, isOverwritten is the optimistic locking token. + assert.strictEqual(created.__rerum.generatedBy, AGENT, 'attribution cannot be forged') + assert.strictEqual(created.__rerum.isReleased, '', 'a client cannot mint a pre-released object') + assert.strictEqual(created.__rerum.isOverwritten, '', 'a client cannot mint a locking token') + + // An external object imported through an update is also a root, but it remembers its external self. + const imported = utils.configureRerumOptions(AGENT, { '@id': 'https://elsewhere.example.org/thing', __rerum: structuredClone(FORGED) }, false, true) + assert.strictEqual(imported.__rerum.history.prime, 'root') + assert.strictEqual(imported.__rerum.history.previous, 'https://elsewhere.example.org/thing') + assert.strictEqual(imported.__rerum.releases.previous, '') + }) + + it('carries the version and release chain forward when updating an existing object', () => { + const fromRoot = utils.configureRerumOptions( + AGENT, + { '@id': RECEIVED_ID, __rerum: { history: { prime: 'root', previous: '', next: [] } } }, + true, + false + ) + assert.strictEqual(fromRoot.__rerum.history.prime, RECEIVED_ID, 'the root object cannot pass "root" on as the prime') + assert.strictEqual(fromRoot.__rerum.history.previous, RECEIVED_ID) + + const PRIME = 'https://store.rerum.io/v1/id/prime-id' + const RELEASE = 'https://store.rerum.io/v1/id/released-id' + const fromDescendant = utils.configureRerumOptions( + AGENT, + { + '@id': RECEIVED_ID, + __rerum: { + history: { prime: PRIME, previous: 'https://store.rerum.io/v1/id/older-id', next: [] }, + releases: { previous: RELEASE, next: [], replaces: '' } + } + }, + true, + false + ) + assert.strictEqual(fromDescendant.__rerum.history.prime, PRIME, 'an object that knows its prime passes it on') + assert.strictEqual(fromDescendant.__rerum.history.previous, RECEIVED_ID) + assert.strictEqual(fromDescendant.__rerum.releases.previous, RELEASE) + }) }) describe('controllers/utils.js generateSlugId', () => { @@ -212,6 +270,12 @@ describe('controllers/utils.js _contextid', () => { _contextid(['http://example.com/other', 'http://iiif.io/api/presentation/3/context.json']), true ) + // An inline term definition object, or any other non-string member, names no context. + assert.strictEqual( + _contextid([{ '@vocab': 'http://example.org/terms#' }, 'http://www.w3.org/ns/anno.jsonld']), + true + ) + assert.strictEqual(_contextid([{ '@vocab': 'http://example.org/terms#' }]), false) }) it('returns false for non-string, non-array input', () => { @@ -288,3 +352,73 @@ describe('controllers/utils.js getPagination', () => { assert.ok(result.limit < huge, `limit should be clamped below ${huge}`) }) }) + +describe('controllers/utils.js findLeafAnnotationsFor', () => { + const ENTITY_URI = 'https://store.rerum.io/v1/id/entity-id' + const SLUG_URI = 'https://store.rerum.io/v1/id/entity-slug' + const TARGET_KEYS = ['target', 'target.@id', 'target.id', 'target.source', 'target.source.@id', 'target.source.id'] + + let capturedQuery + + /** + * Point db.find() at a cursor over the given documents and record the filter it was called with. + * + * @param docs The Annotation documents the cursor will yield. + */ + function armFind(docs = []) { + resetMocks() + capturedQuery = undefined + db.find.mockImplementationOnce(query => { + capturedQuery = query + return createCursor(docs) + }) + } + + // $and[0] holds the target conditions, $and[1] the Annotation type conditions. + const targetConditions = () => capturedQuery.$and[0].$or + + it('constrains the query to the leaf versions, every target key, and every type spelling', async () => { + armFind() + + await findLeafAnnotationsFor([ENTITY_URI, SLUG_URI, ENTITY_URI, undefined, '']) + + assert.deepStrictEqual(capturedQuery['__rerum.history.next'], { $exists: true, $size: 0 }) + for (const targetKey of TARGET_KEYS) { + const values = targetConditions() + .filter(condition => Object.hasOwn(condition, targetKey)) + .map(condition => condition[targetKey]) + assert.strictEqual(values.length, 8, `${targetKey}: two URIs, each in two schemes and as a fragment`) + assert.ok(values.includes(ENTITY_URI) && values.includes(ENTITY_URI.replace(/^https/, 'http'))) + assert.ok(values.includes(SLUG_URI), 'every URI the entity answers to is targeted') + const patterns = values.filter(value => value instanceof RegExp) + assert.ok(patterns.some(pattern => pattern.test(`${ENTITY_URI}#xywh=0,0,100,100`)), 'a fragment of the URI is a match') + assert.ok( + patterns.every(pattern => !pattern.test('https://storeXrerum.io/v1/id/entity-id#xywh=0,0,100,100')), + 'the URI is escaped, so its dots are not wildcards' + ) + } + assert.deepStrictEqual(capturedQuery.$and[1].$or, [ + { type: 'Annotation' }, + { type: 'oa:Annotation' }, + { type: 'http://www.w3.org/ns/oa#Annotation' }, + { type: 'https://www.w3.org/ns/oa#Annotation' }, + { '@type': 'Annotation' }, + { '@type': 'oa:Annotation' }, + { '@type': 'http://www.w3.org/ns/oa#Annotation' }, + { '@type': 'https://www.w3.org/ns/oa#Annotation' } + ]) + + armFind() + await findLeafAnnotationsFor('bare-slug') + assert.strictEqual(targetConditions().length, TARGET_KEYS.length, 'a non-URI target has no scheme or fragment to anticipate') + }) + + it('gathers nothing rather than querying on an empty $or when there is no target', async () => { + // An empty '$or' is a MongoDB error, not an empty result, so the query is never sent. + armFind([{ _id: 'anno001', type: 'Annotation' }]) + + assert.deepStrictEqual(await findLeafAnnotationsFor([undefined, '', null]), []) + assert.deepStrictEqual(await findLeafAnnotationsFor(undefined), []) + assert.strictEqual(capturedQuery, undefined, 'nothing to target is nothing to gather') + }) +}) diff --git a/controllers/crud.js b/controllers/crud.js index 758a048a..7d1f90dc 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -6,7 +6,7 @@ */ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js' +import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' /** * Create a new Linked Open Data object in RERUM v1. @@ -127,8 +127,179 @@ const id = async function (req, res, next) { } } +/** + * The expand job always constrains the Annotations it gathers to the leaf versions, to the + * Annotation types, and to the entity in the request URI. A client cannot influence those, so + * these keys are dropped from a supplied filter body by exact name or dotted prefix. + */ +const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"] + +/** + * The Annotation body types whose value is kept whole rather than read as a single assertion. + */ +const TEXTUAL_BODY_TYPES = new Set([ + "TextualBody", "oa:TextualBody", + "http://www.w3.org/ns/oa#TextualBody", "https://www.w3.org/ns/oa#TextualBody" +]) + +/** + * Reduce a supplied POST body to the literal MongoDB filter keys the expand job will honor. + * @param supplied The parsed JSON request body. + * @return An object of filter keys, minus the ones this endpoint owns. + */ +function sanitizeExpansionFilters(supplied) { + const filters = {} + for (const [key, value] of Object.entries(supplied)) { + if (RESERVED_FILTER_KEYS.some(reserved => key === reserved || key.startsWith(`${reserved}.`))) continue + filters[key] = value + } + return filters +} + +/** + * The [key, value] assertions an Annotation makes about the entity it targets. + * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other + * property of the Annotation can leak onto the entity. + * + * Anticipates the likely Annotation body formats + * - bodyValue: 'text' the W3C shorthand, which has no key of its own + * - body: {'key': 'value'} a single assertion + * - body: {'key': {...}} a single assertion, value kept as-is + * - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive + * - body: {'@type':'oa:TextualBody', ...} the 'oa:' prefixed spelling of the same W3C class + * - body: {'type':['TextualBody'], ...} the same body, type serialized as a JSON-LD Array + * - body: [{'key': 'value'}] one body, serialized as a JSON-LD Array + * + * @param anno An Annotation document. + * @return An Array of [key, value] pairs to merge onto the entity. + */ +function assertionsFrom(anno) { + const assertions = [] + if (typeof anno.bodyValue === "string") assertions.push(["bodyValue", anno.bodyValue]) + // In JSON-LD a one-element Array and the bare value are the same body, so unwrap it first. The + // check below is about how many bodies an Annotation carries, not how they were serialized. + const body = Array.isArray(anno.body) && anno.body.length === 1 ? anno.body[0] : anno.body + // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an + // external resource with no embedded value to expand with. + if (Array.isArray(body) || !body || typeof body !== "object") return assertions + const bodyType = body.type ?? body["@type"] + const bodyTypes = Array.isArray(bodyType) ? bodyType : [bodyType] + if (bodyTypes.some(t => TEXTUAL_BODY_TYPES.has(t))) { + assertions.push(["bodyValue", body]) + return assertions + } + const keys = Object.keys(body) + // Any other multi-key body is structural rather than assertional and cannot be attributed to a + // single entity property. This is what skips the Choice, Composite, and List multiplicity constructs. + if (keys.length !== 1) return assertions + assertions.push([keys[0], body[keys[0]]]) + return assertions +} + +/** + * Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values. + * When more than one current Annotation asserts the same key, or the entity already carries it, + * the values collect into an Array, the record's own value first. + * @param primitiveEntity The unexpanded entity. + * @param annoAssertions An Array holding the [key, value] assertions read from each Annotation. + * @return A new, expanded entity object. + */ +function applyExpansionAnnotations(primitiveEntity, annoAssertions) { + const expandedEntity = structuredClone(primitiveEntity) + const rerumProp = expandedEntity.__rerum + delete expandedEntity.__rerum + for (const assertions of annoAssertions) { + for (const [key, value] of assertions) { + if (PROTECTED_EXPANSION_KEYS.has(key)) continue + if (!Object.hasOwn(expandedEntity, key)) { + expandedEntity[key] = value + continue + } + const existing = Array.isArray(expandedEntity[key]) ? expandedEntity[key] : [expandedEntity[key]] + const contributed = Array.isArray(value) ? value : [value] + expandedEntity[key] = [...existing, ...contributed] + } + } + if (rerumProp !== undefined) expandedEntity.__rerum = rerumProp + return expandedEntity +} + +/** + * Query the MongoDB for the object with the _id or __rerum.slug provided in the request URL, then + * merge in the assertions of all the current leaf Annotations targeting it. + * + * GET recognizes the '?generator=' and '?creator=' convenience parameters only. + * POST reads literal MongoDB filter keys from the JSON body and ignores URL parameters as filters. + * Neither method pages. A client asks once and receives the entity assembled. + * */ +const idExpanded = async function (req, res, next) { + res.set("Content-Type", "application/json; charset=utf-8") + const requestedId = req.params["_id"] + const isPost = req.method === "POST" + let filters = {} + if (isPost) { + // Express leaves the body undefined when a POST supplies none. That is an unfiltered expand. + const supplied = req.body ?? {} + if (typeof supplied !== "object" || Array.isArray(supplied)) { + const err = { + "message": "The /expanded request body must be a JSON object of filter properties.", + "status": 400 + } + return next(utils.createExpressError(err)) + } + filters = sanitizeExpansionFilters(supplied) + } + else { + // Repeated query parameters arrive as an Array, which is not a filter value we will apply. + if (typeof req.query.generator === "string" && req.query.generator) filters["__rerum.generatedBy"] = req.query.generator + if (typeof req.query.creator === "string" && req.query.creator) filters.creator = req.query.creator + } + try { + const match = await db.findOne({"$or": [{"_id": requestedId}, {"__rerum.slug": requestedId}]}) + if (!match) { + const err = { + "message": `No RERUM object with id '${requestedId}'`, + "status": 404 + } + return next(utils.createExpressError(err)) + } + const deleted = utils.isDeleted(match) + const targetId = match["@id"] ?? match.id + // an Annotation may target an entity by its Slug instead of by its '@id'. + const slug = match.__rerum?.slug + const lastSlash = targetId?.lastIndexOf("/") ?? -1 + const slugTargetId = slug && lastSlash !== -1 ? targetId.slice(0, lastSlash + 1) + slug : undefined + // Read off the record while it is still whole. idNegotiation() below alters it in place. + const currentVersion = match.__rerum?.isOverwritten ?? "" + const annos = deleted ? [] : await findLeafAnnotationsFor([targetId, slugTargetId], filters) + // Every leaf Annotation matching the filter is gathered. This is the count. + res.set('Annotations-Gathered', String(annos.length)) + // How many of the Annotations contribute an assertion. May be less than annotations gathered. + // The count reports the Annotations that actually alter the entity. + const merged = annos + .map(anno => assertionsFrom(anno).filter(([key]) => !PROTECTED_EXPANSION_KEYS.has(key))) + .filter(assertions => assertions.length > 0) + res.set('Annotations-Merged', String(merged.length)) + // Negotiate first, so identity is settled from the record's own '@context' before anything is merged. + const negotiated = idNegotiation(match) + const identity = _contextid(negotiated["@context"]) ? negotiated.id : negotiated["@id"] + const expanded = deleted ? negotiated : applyExpansionAnnotations(negotiated, merged) + // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). + if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate") + // Headers describe the stored record, so they match GET /v1/id/:_id for the same record. + res.set(utils.configureWebAnnoHeadersFor(negotiated)) + // Include current version for optimistic locking + res.set('Current-Overwritten-Version', currentVersion) + res.location(identity) + res.json(expanded) + } catch (error) { + return next(utils.createExpressError(error)) + } +} + export { create, query, - id + id, + idExpanded } diff --git a/controllers/gog.js b/controllers/gog.js index db69a3e5..778658a7 100644 --- a/controllers/gog.js +++ b/controllers/gog.js @@ -6,9 +6,9 @@ * @author cubap, thehabes */ -import { newID, isValidID, db } from '../database/index.js' +import { db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation } from './utils.js' +import { _contextid, getAgentClaim, getPagination, idNegotiation, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' // The Gallery of Glosses agents, by RERUM ObjectId. Prod (store) and dev (devstore) mint different // agents; only the trailing id is compared, so either host spelling matches. @@ -312,20 +312,6 @@ const _gog_glosses_from_manuscript = async function (req, res, next) { * Find relevant Annotations targeting a primitive RERUM entity. This is a 'full' expand. * Add the descriptive information in the Annotation bodies to the primitive object. * -* Anticipate likely Annotation body formats -* - anno.body -* - anno.body.value -* -* Anticipate likely Annotation target formats -* - target: 'uri' -* - target: {'id':'uri'} -* - target: {'@id':'uri'} -* -* Anticipate likely Annotation type formats -* - {"type": "Annotation"} -* - {"@type": "Annotation"} -* - {"@type": "oa:Annotation"} -* * @param primitiveEntity - An existing RERUM object * @param GENERATOR - A registered RERUM app's User Agent * @param CREATOR - Some kind of string representing a specific user. Often combined with GENERATOR. @@ -336,77 +322,26 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde // An entity is expandable if it carries a URI under either '@id' or 'id'. if(!primitiveEntity?.["@id"] && !primitiveEntity?.id) return primitiveEntity const targetId = primitiveEntity["@id"] ?? primitiveEntity.id ?? "unknown" - // '$and' is always present so the GENERATOR and CREATOR blocks below can push into it from - // either branch. 'annoTypeConditions' is always pushed, so it is never the empty Array Mongo rejects. - let queryObj = { - "__rerum.history.next": { $exists: true, $size: 0 }, - "$and": [] - } - let targetPatterns = ["target", "target.@id", "target.id"] - let targetConditions = [] - let annoTypeConditions = [{"type": "Annotation"}, {"@type":"Annotation"}, {"@type":"oa:Annotation"}] - - if (targetId.startsWith("http")) { - for(const targetKey of targetPatterns){ - targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "http") }) - targetConditions.push({ [targetKey]: targetId.replace(/^https?/, "https") }) - } - queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) - } - else{ - queryObj["$and"].push({"$or": annoTypeConditions}) - queryObj.target = targetId - } - - // Only expand with data from a specific app - if(GENERATOR) { - // Need to check http:// and https:// - const generatorConditions = [ - {"__rerum.generatedBy": GENERATOR.replace(/^https?/, "http")}, - {"__rerum.generatedBy": GENERATOR.replace(/^https?/, "https")} - ] - if (GENERATOR.startsWith("http")) { - queryObj["$and"].push({"$or": generatorConditions }) - } - else{ - // It should be a URI, but this can be a fallback. - queryObj["__rerum.generatedBy"] = GENERATOR - } - } - // Only expand with data from a specific creator - if(CREATOR) { - // Need to check http:// and https:// - const creatorConditions = [ - {"creator": CREATOR.replace(/^https?/, "http")}, - {"creator": CREATOR.replace(/^https?/, "https")} - ] - if (CREATOR.startsWith("http")) { - queryObj["$and"].push({"$or": creatorConditions }) - } - else{ - // It should be a URI, but this can be a fallback. - queryObj["creator"] = CREATOR - } - } - - // Get the Annotations targeting this Entity from the db. Remove _id property. - // Assuming we do not need paged query here - let matches = await db.find(queryObj).toArray() - matches = matches.map(o => { - delete o._id - return o - }) + const filters = {} + if(GENERATOR) filters["__rerum.generatedBy"] = GENERATOR + if(CREATOR) filters.creator = CREATOR + const matches = await findLeafAnnotationsFor(targetId, filters) // Combine the Annotation bodies with the primitive object. - // Mirror DEER's client-side expand() (deer-utils.js buildValueObject) // When more than one current Annotation asserts the same key, collect the values into an Array. let expandedEntity = structuredClone(primitiveEntity) + const rerumProp = expandedEntity.__rerum + delete expandedEntity.__rerum for(const anno of matches){ - const body = anno.body - if(!body || typeof body !== "object") continue + // In JSON-LD a one-element Array and the bare value are the same body, so unwrap it first. + const body = Array.isArray(anno.body) && anno.body.length === 1 ? anno.body[0] : anno.body + // Annotations carrying multiple bodies are not expanded with. + if(!body || typeof body !== "object" || Array.isArray(body)) continue const keys = Object.keys(body) if(keys.length !== 1) continue const key = keys[0] + // An Annotation body cannot overwrite the entity's identity or its system properties. + if(PROTECTED_EXPANSION_KEYS.has(key)) continue const assertion = body[key] const valueObject = { value: assertion?.value ?? assertion, @@ -417,7 +352,7 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde }, evidence: assertion?.evidence ?? anno.evidence ?? "" } - if(expandedEntity.hasOwnProperty(key)){ + if(Object.hasOwn(expandedEntity, key)){ expandedEntity[key] = Array.isArray(expandedEntity[key]) ? [...expandedEntity[key], valueObject] : [expandedEntity[key], valueObject] @@ -426,6 +361,7 @@ const expand = async function(primitiveEntity, GENERATOR=undefined, CREATOR=unde expandedEntity[key] = valueObject } } + if(rerumProp !== undefined) expandedEntity.__rerum = rerumProp return expandedEntity } @@ -445,23 +381,32 @@ const expandedId = async function (req, res, next) { err = Object.assign(err, { message: `No RERUM object with id '${id}'`, status: 404 }) return next(utils.createExpressError(err)) } + // Deleted objects don't have a generator to match on, just return the tombstone no matter what. + if (utils.isDeleted(match)) { + res.set(utils.configureWebAnnoHeadersFor(match)) + res.set("Cache-Control", "max-age=86400, must-revalidate") + res.set("Current-Overwritten-Version", "") + const tombstone = idNegotiation(match) + res.location(_contextid(tombstone["@context"]) ? tombstone.id : tombstone["@id"]) + return res.json(tombstone) + } const generator = match.__rerum?.generatedBy const agentID = generator?.split("/").pop() if (!GOG_AGENTS.includes(agentID)) { err = Object.assign(err, { - message: `This request can only be made for Gallery of Glosses generated data.`, - status: 403 + message: `No Gallery of Glosses record with id '${id}'. This URI serves GoG generated data only.`, + status: 404 }) return next(utils.createExpressError(err)) } + let expanded = await expand(match, generator) + expanded = idNegotiation(expanded) // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h). - res.set(utils.configureWebAnnoHeadersFor(match)) res.set("Cache-Control", "max-age=86400, must-revalidate") - // No Last-Modified here, unlike GET /v1/id/:_id. It would compare against the root entity - // before expand() merges the targeting Annotations. + // Headers describe the stored record, so they match GET /v1/id/:_id for the same record. + res.set(utils.configureWebAnnoHeadersFor(match)) + // Include current version for optimistic locking res.set("Current-Overwritten-Version", match.__rerum?.isOverwritten ?? "") - let expanded = await expand(match, generator) - expanded = idNegotiation(expanded) res.location(_contextid(expanded["@context"]) ? expanded.id : expanded["@id"]) res.json(expanded) } catch (error) { diff --git a/controllers/utils.js b/controllers/utils.js index dd455d05..a5c09bc5 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -30,7 +30,7 @@ function getPagination(query = {}, defaultLimit = 100) { /** * Check if a @context value contains a known @id-id mapping context * - * @param contextInput An Array of string URIs or a string URI. + * @param contextInput A string URI, or an Array of them. * @return A boolean */ function _contextid(contextInput) { @@ -46,6 +46,8 @@ function _contextid(contextInput) { ] if(Array.isArray(contextInput)) { for(const c of contextInput) { + // An inline term definition object, or any other non-string member, names no context. + if(typeof c !== "string") continue contextURI = c bool = knownContexts.some(contextCheck) if(bool) break @@ -70,14 +72,11 @@ const idNegotiation = function (resBody) { const _id = resBody._id delete resBody._id if(!resBody["@context"]) return resBody - let modifiedResBody = structuredClone(resBody) + if(!_contextid(resBody["@context"])) return structuredClone(resBody) const context = { "@context": resBody["@context"] } - if(_contextid(resBody["@context"])) { - delete resBody["@id"] - delete resBody["@context"] - modifiedResBody = Object.assign(context, { "id": process.env.RERUM_ID_PREFIX + _id }, resBody) - } - return modifiedResBody + delete resBody["@id"] + delete resBody["@context"] + return Object.assign(context, { "id": process.env.RERUM_ID_PREFIX + _id }, resBody) } /** @@ -109,6 +108,125 @@ const generateSlugId = async function(slug_id="", next){ return slug_return } +/** + * RERUM has minted these two under both 'http' and 'https' over the years, so a filter on either + * must match both spellings. Every other supplied filter key is applied exactly as given. + */ +const URI_DOUBLED_FILTER_KEYS = new Set(["__rerum.generatedBy", "creator"]) + +/** + * The properties an Annotation can carry the URI of its target under. + * Choice, Composite, and List target constructs, whose members sit in an 'items' Array. They are not supported here. + */ +const TARGET_KEYS = ["target", "target.@id", "target.id", "target.source", "target.source.@id", "target.source.id"] + +/** + * Identity, system, and processing properties an Annotation body must never overwrite when its + * assertions are merged onto an entity. + */ +const PROTECTED_EXPANSION_KEYS = new Set(["@id", "id", "_id", "__rerum", "__deleted", "__proto__", "@context"]) + +/** + * Escape the RegExp metacharacters in a literal so it can be embedded in a pattern and match only + * itself. A RERUM URI has at least the dots of its host to escape. + * @param literal A string to be matched literally. + * @return The same string, safe to concatenate into a RegExp source. + */ +function escapeRegex(literal) { + return literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +} + +/** + * Find the current (leaf) Annotations targeting an entity, for expansion. + * + * Anticipates the likely Annotation target formats + * - target: 'uri' + * - target: {'id':'uri'} + * - target: {'@id':'uri'} + * - target: {'source':'uri', 'type':'SpecificResource'} the W3C SpecificResource + * - target: {'source':{'id':'uri'}} a SpecificResource with an embedded source + * - target: 'uri#xywh=0,0,100,100' a fragment of the resource + * and the likely Annotation type formats + * - {"type": "Annotation"}, {"type": "oa:Annotation"}, {"type": "http(s)://www.w3.org/ns/oa#Annotation"} + * - {"@type": "Annotation"}, {"@type": "oa:Annotation"}, {"@type": "http(s)://www.w3.org/ns/oa#Annotation"} + * + * Only 'target' and 'body' are read, whatever the type spelling says. + * + * Any entity can answer to more than one URI because of Slugs Every match is gathered, in a single cursor. + * The result is sorted by '_id' before it is returned. That is roughly Annotation creation order, and it makes + * the expansion reproducible. + * + * @param targetIds The '@id' or 'id' URI of the entity being expanded, or an Array of the URIs it + * is known by when it answers to more than one. + * @param filters Literal MongoDB filter keys to AND into the query. + * @return An Array of every matching Annotation document sorted by '_id', with '_id' removed. + */ +const findLeafAnnotationsFor = async function (targetIds, filters = {}) { + const EXPANSION_BATCH_SIZE = 200 + const targetURIs = [...new Set((Array.isArray(targetIds) ? targetIds : [targetIds]).filter(Boolean))] + // Nothing to target is nothing to gather. An empty '$or' is a MongoDB error, not an empty result. + if (targetURIs.length === 0) return [] + // '$and' is always present so the target, type, and filter conditions can push into it. + const queryObj = { + "__rerum.history.next": { $exists: true, $size: 0 }, + "$and": [] + } + const annoTypeConditions = [ + {"type": "Annotation"}, {"type": "oa:Annotation"}, + {"type": "http://www.w3.org/ns/oa#Annotation"}, {"type": "https://www.w3.org/ns/oa#Annotation"}, + {"@type": "Annotation"}, {"@type": "oa:Annotation"}, + {"@type": "http://www.w3.org/ns/oa#Annotation"}, {"@type": "https://www.w3.org/ns/oa#Annotation"} + ] + // Every URI the entity answers to contributes its conditions to the same '$or'. + const targetConditions = [] + for (const targetURI of targetURIs) { + if (!/^https?:\/\//.test(targetURI)) { + // Not a URI, so there is no http/https spelling or fragment of it to anticipate, but it + // can still sit under any of the target keys. + for (const targetKey of TARGET_KEYS) targetConditions.push({ [targetKey]: targetURI }) + continue + } + // Hanging a fragment off the URI is the other W3C way to target part of a resource rather + // than the whole of it, and an exact match will not catch one. + const fragmentPatterns = ["http", "https"].map(scheme => + new RegExp(`^${escapeRegex(targetURI.replace(/^https?/, scheme))}#`) + ) + // 'target.source' is the W3C SpecificResource, which is how an Annotation targets a + // selected region of a resource rather than the whole of it. + for (const targetKey of TARGET_KEYS) { + targetConditions.push({ [targetKey]: targetURI.replace(/^https?/, "http") }) + targetConditions.push({ [targetKey]: targetURI.replace(/^https?/, "https") }) + for (const fragmentPattern of fragmentPatterns) targetConditions.push({ [targetKey]: fragmentPattern }) + } + } + queryObj["$and"].push({"$or": targetConditions}, {"$or": annoTypeConditions}) + for (const [key, value] of Object.entries(filters)) { + if (URI_DOUBLED_FILTER_KEYS.has(key) && typeof value === "string" && /^https?:\/\//.test(value)) { + queryObj["$and"].push({"$or": [ + { [key]: value.replace(/^https?/, "http") }, + { [key]: value.replace(/^https?/, "https") } + ]}) + continue + } + queryObj["$and"].push({ [key]: value }) + } + const matches = [] + // One cursor, paged server-side by the driver. The cursor transfers in the same stride and reads each document once. + for await (const anno of db.find(queryObj).batchSize(EXPANSION_BATCH_SIZE)) { + matches.push(anno) + } + // The query plan promises no order, so sort before '_id' is dropped. The same data then always + // assembles into the same entity, which keeps the ETag stable and lets a revalidation answer 304. + matches.sort((a, b) => { + const left = String(a._id) + const right = String(b._id) + if (left < right) return -1 + return left > right ? 1 : 0 + }) + for (const anno of matches) delete anno._id + return matches +} + // Handle index actions const index = function (req, res, next) { res.json({ @@ -465,6 +583,8 @@ async function healReleasesTree(releasing) { export { _contextid, idNegotiation, + findLeafAnnotationsFor, + PROTECTED_EXPANSION_KEYS, getPagination, generateSlugId, index, diff --git a/database/__mocks__/index.js b/database/__mocks__/index.js index 51ec6a2c..886842ba 100644 --- a/database/__mocks__/index.js +++ b/database/__mocks__/index.js @@ -39,12 +39,26 @@ function createMockFunction(implementation = () => undefined) { return fn } -function createCursor() { - return { +/** + * A stand-in for the driver's FindCursor. The chainable methods return the cursor the way the + * driver's do, and the cursor is async-iterable over whatever toArray() resolves to so a caller + * can read it either way. + * + * @param docs The documents this cursor yields. + * @return A cursor double. + */ +export function createCursor(docs = []) { + const cursor = { limit: createMockFunction(function () { return this }), skip: createMockFunction(function () { return this }), - toArray: createMockFunction(() => Promise.resolve([])) + batchSize: createMockFunction(function () { return this }), + toArray: createMockFunction(() => Promise.resolve(docs)), + async *[Symbol.asyncIterator]() { + // A cursor whose toArray was reset has no implementation left, so it iterates as empty. + for (const doc of await cursor.toArray() ?? []) yield doc + } } + return cursor } const defaultBulkWriteResponse = () => ({ diff --git a/db-controller.js b/db-controller.js index 7f161667..99f5c163 100644 --- a/db-controller.js +++ b/db-controller.js @@ -8,7 +8,7 @@ // Import controller modules import { index, idNegotiation, generateSlugId, remove } from './controllers/utils.js' -import { create, query, id } from './controllers/crud.js' +import { create, query, id, idExpanded } from './controllers/crud.js' import { searchAsWords, searchAsPhrase } from './controllers/search.js' import { deleteObj } from './controllers/delete.js' import { putUpdate, patchUpdate, patchSet, patchUnset, overwrite } from './controllers/update.js' @@ -32,6 +32,7 @@ export default { searchAsWords, searchAsPhrase, id, + idExpanded, bulkCreate, bulkUpdate, queryHeadRequest, diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index 7174b34a..b5464689 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -28,6 +28,15 @@ paths: responses: '200': description: Object payload + headers: + Cache-Control: + $ref: '#/components/headers/RecordCacheControl' + Last-Modified: + $ref: '#/components/headers/RecordLastModified' + Location: + $ref: '#/components/headers/CanonicalLocation' + Current-Overwritten-Version: + $ref: '#/components/headers/CurrentOverwrittenVersion' content: application/json: schema: @@ -42,8 +51,112 @@ paths: responses: '200': description: Object headers + headers: + Cache-Control: + $ref: '#/components/headers/RecordCacheControl' + Last-Modified: + $ref: '#/components/headers/RecordLastModified' + Location: + $ref: '#/components/headers/CanonicalLocation' + Current-Overwritten-Version: + $ref: '#/components/headers/CurrentOverwrittenVersion' '404': $ref: '#/components/responses/NotFound' + /id/{id}/expanded: + get: + summary: Read object by id with its current Annotations merged in + operationId: getExpandedObjectById + parameters: + - $ref: '#/components/parameters/ObjectId' + - $ref: '#/components/parameters/ExpansionGenerator' + - $ref: '#/components/parameters/ExpansionCreator' + responses: + '200': + description: >- + Expanded object payload. A deleted record is returned as stored, with no expansion. + headers: + Annotations-Gathered: + $ref: '#/components/headers/AnnotationsGathered' + Annotations-Merged: + $ref: '#/components/headers/AnnotationsMerged' + Cache-Control: + $ref: '#/components/headers/ExpansionCacheControl' + Location: + $ref: '#/components/headers/CanonicalLocation' + Current-Overwritten-Version: + $ref: '#/components/headers/CurrentOverwrittenVersion' + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + '404': + $ref: '#/components/responses/NotFound' + '405': + description: Method not allowed — this endpoint only permits GET, HEAD, and POST. + head: + summary: Read expanded object headers by id + operationId: headExpandedObjectById + parameters: + - $ref: '#/components/parameters/ObjectId' + - $ref: '#/components/parameters/ExpansionGenerator' + - $ref: '#/components/parameters/ExpansionCreator' + responses: + '200': + description: >- + Expanded object headers. A deleted record is returned as stored, with no expansion. + headers: + Annotations-Gathered: + $ref: '#/components/headers/AnnotationsGathered' + Annotations-Merged: + $ref: '#/components/headers/AnnotationsMerged' + Cache-Control: + $ref: '#/components/headers/ExpansionCacheControl' + Location: + $ref: '#/components/headers/CanonicalLocation' + Current-Overwritten-Version: + $ref: '#/components/headers/CurrentOverwrittenVersion' + '404': + $ref: '#/components/responses/NotFound' + '405': + description: Method not allowed — this endpoint only permits GET, HEAD, and POST. + post: + summary: Read object by id with its current Annotations merged in, filtered by the request body + operationId: postExpandedObjectById + parameters: + - $ref: '#/components/parameters/ObjectId' + requestBody: + required: false + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + responses: + '200': + description: >- + Expanded object payload. A deleted record is returned as stored, with no expansion. + headers: + Annotations-Gathered: + $ref: '#/components/headers/AnnotationsGathered' + Annotations-Merged: + $ref: '#/components/headers/AnnotationsMerged' + Location: + $ref: '#/components/headers/CanonicalLocation' + Current-Overwritten-Version: + $ref: '#/components/headers/CurrentOverwrittenVersion' + content: + application/json: + schema: + $ref: '#/components/schemas/GenericObject' + '400': + $ref: '#/components/responses/BadRequest' + '404': + $ref: '#/components/responses/NotFound' + '405': + description: Method not allowed — this endpoint only permits GET, HEAD, and POST. + '413': + $ref: '#/components/responses/PayloadTooLarge' + '415': + $ref: '#/components/responses/UnsupportedMediaType' /since/{id}: get: summary: Read updates since id @@ -530,6 +643,52 @@ paths: '409': $ref: '#/components/responses/Conflict' components: + headers: + AnnotationsGathered: + description: >- + How many current Annotations targeting the entity were gathered for this expansion. + schema: + type: integer + minimum: 0 + AnnotationsMerged: + description: >- + How many of the gathered Annotations could contribute an assertion. + schema: + type: integer + minimum: 0 + ExpansionCacheControl: + description: >- + Browser caching policy for the expanded entity. An entity's Annotations are volatile while it + is first created and stable afterward, so the expansion is cacheable for 24 hours. + schema: + type: string + example: max-age=86400, must-revalidate + RecordCacheControl: + description: >- + Browser caching policy for the record. A record is only replaced by an update, which mints a + new URI, so the response at this URI is cacheable for 24 hours. + schema: + type: string + example: max-age=86400, must-revalidate + RecordLastModified: + description: >- + The record's '__rerum.isOverwritten' time, or its '__rerum.createdAt' time when it has never + been overwritten, so a client can revalidate with 'If-Modified-Since'. + schema: + type: string + format: date-time + CanonicalLocation: + description: >- + The canonical URI of the record, in its '_id' form. A request made by Slug is answered with + the stable '_id' URI here rather than the Slug it was asked for. + schema: + type: string + CurrentOverwrittenVersion: + description: >- + The record's '__rerum.isOverwritten' value, for optimistic locking. Empty when the record has + never been overwritten. + schema: + type: string parameters: ObjectId: in: path @@ -537,6 +696,20 @@ components: required: true schema: type: string + ExpansionGenerator: + in: query + name: generator + required: false + description: Only expand with Annotations generated by this registered app agent. + schema: + type: string + ExpansionCreator: + in: query + name: creator + required: false + description: Only expand with Annotations attributed to this creator. + schema: + type: string schemas: GenericObject: type: object diff --git a/public/API.html b/public/API.html index 38f2df93..3cd1a6a6 100644 --- a/public/API.html +++ b/public/API.html @@ -41,13 +41,14 @@

API (1.1.0)