diff --git a/packages/base/card-serialization.ts b/packages/base/card-serialization.ts index dc959012c8..df242120d3 100644 --- a/packages/base/card-serialization.ts +++ b/packages/base/card-serialization.ts @@ -66,6 +66,11 @@ export interface JSONAPISingleResourceDocument { export interface SerializeOpts { includeComputeds?: boolean; + // Include link fields the card does not actually have. By default + // serialization keeps only the relationships present in the card's data — a + // set target or an authored empty `{ self: null }` — plus contained fields + // (always present); set this to serialize every declared relationship, + // including never-authored ones (as `{ self: null }`). includeUnrenderedFields?: boolean; useAbsoluteURL?: boolean; omitFields?: [typeof BaseDef]; diff --git a/packages/base/field-support.ts b/packages/base/field-support.ts index 3d02cb707c..bf9fd4db96 100644 --- a/packages/base/field-support.ts +++ b/packages/base/field-support.ts @@ -194,7 +194,15 @@ export function getter( return deserialized.get(field.name); } let value = field.emptyValue(instance); - deserialized.set(field.name, value); + // Don't persist a nullish empty value (an unset `linksTo`): writing it would + // fabricate a bucket entry for a link the caller merely read, making an + // unset link indistinguishable from an authored `{ self: null }` when the + // card serializes. Recomputing the nullish empty on the next read is free; + // identity only matters for the mutable empties (arrays / composite fields), + // which are non-null and still persist for reactivity. + if (value != null) { + deserialized.set(field.name, value); + } return value; } } @@ -350,13 +358,14 @@ export function getFieldOverrides( // per serialize, per searchDoc, per getDeps/findInstances recursion — and // `computeFields` walks the prototype chain and resolves every field on each // call. The result is determined by the prototype chain plus, for an instance, -// its polymorphic field overrides (and, with `usedLinksToFieldsOnly`, its -// populated field set). A read-only render only ever GROWS those, so their -// sizes are a sound validity token: an unchanged token means an identical -// result. Gated to the render context so the live app — where instances mutate -// freely (same-size override swaps, field clears) — is never served a memoized -// map. Keyed on the instance/class via a WeakMap so entries fall away with -// their subjects; the token guards reuse across passes. +// its polymorphic field overrides and — under `usedLinksToFieldsOnly` — the set +// of links it has (authored or set): a link joins the map as its data-bucket +// entry appears. A read-only render only ever GROWS both, so their sizes are a +// sound validity token: an unchanged token means an identical result. Gated to +// the render context so the live app — where +// instances mutate freely (same-size override swaps, field clears) — is never +// served a memoized map. Keyed on the instance/class via a WeakMap so entries +// fall away with their subjects; the token guards reuse across passes. const renderFieldsCache = new WeakMap< object, Map< @@ -380,8 +389,14 @@ function renderFieldsCacheToken( } let overrideSize = fieldOverrides.get(subject as BaseDef)?.size ?? 0; if (!usedLinksToFieldsOnly) { + // The full field map lists every declared field, so it depends only on the + // prototype chain and polymorphic overrides — not on what the instance set. return `o${overrideSize}`; } + // A link is kept only when the card actually has it (authored or set), so + // this map — unlike the full map — also depends on the instance's data. The + // data bucket only grows within a render pass, so its size is a sound + // validity token for that dependency. let usedSize = deserializedData.get(subject as BaseDef)?.size ?? 0; return `o${overrideSize}:u${usedSize}`; } @@ -432,11 +447,11 @@ function computeFields( opts?: { usedLinksToFieldsOnly?: boolean; includeComputeds?: boolean }, ): { [fieldName: string]: Field } { let obj: object | null; - let usedFields: string[] = []; + let instance: BaseDef | undefined; if (isCardOrField(cardInstanceOrClass)) { // this is a card instance + instance = cardInstanceOrClass; obj = Reflect.getPrototypeOf(cardInstanceOrClass); - usedFields = getUsedFields(cardInstanceOrClass); } else { // this is a card class obj = (cardInstanceOrClass as typeof BaseDef).prototype; @@ -460,9 +475,15 @@ function computeFields( maybeField.computeVia || !['contains', 'containsMany'].includes(maybeField.fieldType) ) { + // `usedLinksToFieldsOnly` keeps only the link fields the card actually + // has — an authored empty (`{ self: null }`) or a set target — and drops + // never-authored links. This is a property of the card's data, not of + // searchability (searchable annotations drive the search doc, not this + // serialization). Callers wanting every declared link pass + // `includeUnrenderedFields`; contained fields are always kept. if ( opts?.usedLinksToFieldsOnly && - !usedFields.includes(maybeFieldName) && + !isFieldUsed(instance, maybeFieldName) && !['contains', 'containsMany'].includes(maybeField.fieldType) ) { return []; @@ -483,10 +504,35 @@ function computeFields( return fields; } -function getUsedFields(instance: BaseDef): string[] { - // getDataBucket always returns a Map (it creates one if absent), so the spread - // is safe without optional chaining. - return [...getDataBucket(instance).keys()]; +// Whether a link field is "used" for `usedLinksToFieldsOnly` serialization: the +// card actually has the relationship. A link enters the data bucket only when +// it was authored (deserialized from the source — a set target, or an empty +// `{ self: null }` that lands as a `null` entry) or set on the instance; a +// never-authored `linksTo` has no bucket entry, because the getter skips +// persisting its nullish empty value, so a mere render can't mark it "used". +// A `linksToMany` getter must persist its (mutable) backing array even when +// empty, so an empty array is NOT a "had" relationship — reading an unset +// plural link would otherwise fabricate one. This is pure card data, +// independent of searchability. Contained fields never reach here (always kept). +function isFieldUsed( + instance: BaseDef | undefined, + fieldName: string, +): boolean { + if (!instance) { + return false; + } + let bucket = getDataBucket(instance); + if (!bucket.has(fieldName)) { + return false; + } + let value = bucket.get(fieldName); + // An empty backing array is a `linksToMany` the card doesn't actually have + // (never authored, or materialized empty on read); `null` is an authored + // empty `linksTo` and is kept. + if (Array.isArray(value)) { + return value.length > 0; + } + return true; } export function isArrayOfCardOrField( @@ -807,7 +853,8 @@ export interface BrokenLinkFinding { // planted a sentinel there), and an unmaterialized field holds neither a broken // link nor a nested card — so skipping absent fields loses nothing and avoids // the getter's side effect of initializing them with `emptyValue` (which would -// pollute `getUsedFields` / serialization). Relationship state is then read +// pollute the data-bucket-driven `isFieldUsed` signal / serialization). +// Relationship state is then read // through `getRelationshipMembershipState`, which never triggers `lazilyLoadLink`, so a // recursed value surfaces only states that genuinely failed during this render. // `'present'`, `'not-loaded'`, and `'not-set'` are not terminal failures; a @@ -859,19 +906,6 @@ export function getBrokenLinks( ); for (let entry of membership ?? []) { if (entry.kind === 'error' || entry.kind === 'not-found') { - // DIAGNOSTIC LOGGING (CS-11221) — remove after CI passes. Read - // only fields that won't initialize bucket entries via the - // field getter (constructor name + the entry's own reference); - // reading `instance.id` here would write the `id` field's - // emptyValue into the bucket and violate the pure-read contract - // the surrounding `getBrokenLinks` upholds. - console.error('[CS-11221 DIAG] getBrokenLinks finding', { - ownerType: instance?.constructor?.name, - fieldName, - fieldType: field.fieldType, - kind: entry.kind, - reference: entry.reference, - }); findings.push({ fieldName, kind: entry.kind, diff --git a/packages/base/searchable.ts b/packages/base/searchable.ts index 9429ddd8af..e3ceb6fd70 100644 --- a/packages/base/searchable.ts +++ b/packages/base/searchable.ts @@ -6,6 +6,7 @@ import { primitive, relativeTo, routesForField, + type SerializedError, } from '@cardstack/runtime-common'; import { getDataBucket, @@ -15,6 +16,8 @@ import { isNonPresentLink, isNotLoadedValue, peekAtField, + type LinkErrorValue, + type LinkNotFoundValue, } from './field-support'; import { createFromSerialized, @@ -81,36 +84,73 @@ function seedSearchableRoutes(cardClass: typeof BaseDef): string[] { return routes; } +// Outcome of loading a searchable link target: the deserialized card, or the +// terminal sentinel a failed load resolves to. A broken searchable link is +// captured as `{ id }` in the doc AND recorded so the caller can plant the +// sentinel on the owner's field — the diagnostic `getBrokenLinks` reads. +type SearchableTargetResult = + | { status: 'loaded'; card: CardDef } + | { status: 'broken'; sentinel: LinkErrorValue | LinkNotFoundValue }; + +// Build the terminal sentinel a broken link resolves to, mirroring the shape +// `lazilyLoadLink` plants: HTTP 404 → `link-not-found`, anything else → +// `link-error`. `reference` is the resolved (absolute) target url. +function brokenLinkSentinel( + reference: string, + err: unknown, +): LinkErrorValue | LinkNotFoundValue { + let status = Number((err as { status?: unknown })?.status); + let message = + (err as { message?: unknown })?.message != null + ? String((err as { message?: unknown }).message) + : `unable to load ${reference}`; + let isMissing = + status === 404 || /not found/i.test(message) || /missing/i.test(message); + let errorDoc: SerializedError = { + status: Number.isFinite(status) ? status : isMissing ? 404 : 500, + title: isMissing ? 'Link Not Found' : 'Link Error', + message, + additionalErrors: null, + }; + return isMissing + ? { type: 'link-not-found', reference, errorDoc } + : { type: 'link-error', reference, errorDoc }; +} + // Targeted load of a link target by reference: reuse a fully-deserialized // resident instance when present, else load + deserialize the document (the // same load path the lazy link getter uses). The store load is not itself // dependency-tracked; callers record each expanded target in the `dependencies` -// set instead. Returns undefined if the target errors. +// set instead. A missing / broken target resolves to a terminal sentinel; the +// store may surface that either as a returned `CardError` or a thrown rejection +// (e.g. a 404 / invalid-URL on the load path), so both are captured. async function loadSearchableTarget( store: CardStore, reference: string, -): Promise { +): Promise { let resident = store.getCard(reference); if (resident && (resident as any)[isSavedInstance] === true) { - return resident; + return { status: 'loaded', card: resident }; } - // A missing / broken target (or one that can't be loaded) degrades to - // `{ id }` upstream. The - // store may surface that either as a returned `CardError` or a thrown - // rejection (e.g. a 404 / invalid-URL on the load path), so guard both. try { let cardDoc = await store.loadCardDocument(reference); if (isCardError(cardDoc)) { - return undefined; + return { + status: 'broken', + sentinel: brokenLinkSentinel(reference, cardDoc), + }; } - return (await createFromSerialized( - cardDoc.data, - cardDoc, - cardDoc.data.id!, - { store }, - )) as CardDef; - } catch { - return undefined; + return { + status: 'loaded', + card: (await createFromSerialized( + cardDoc.data, + cardDoc, + cardDoc.data.id!, + { store }, + )) as CardDef, + }; + } catch (err) { + return { status: 'broken', sentinel: brokenLinkSentinel(reference, err) }; } } @@ -227,6 +267,7 @@ async function searchableQueryableValue( entries.push([ fieldName, await searchableLink( + value, field!, rawValue, matched, @@ -264,6 +305,7 @@ async function searchableQueryableValue( // broken / cannot be loaded); the expanded declared-type target when a route names // it. Mirrors `LinksTo.queryableValue` + the `{ id }` sentinel handling. async function searchableLink( + owner: any, field: Field, rawValue: any, matched: boolean, @@ -277,8 +319,17 @@ async function searchableLink( return null; } // A broken / not-found link can't be expanded — keep its reference as `{ id }`. + // A searchable target stays a dependency even when broken: recording it + // reindexes the card (clearing the `{ id }` / brokenLinks diagnostic) once the + // target becomes reachable, mirroring the successful-expansion dep below. This + // branch also covers a prior settle pass having planted the sentinel, so the + // authoritative generation reaches it here instead of the load branch below. if (isLinkError(rawValue) || isLinkNotFound(rawValue)) { - return { id: makeAbsoluteURL(rawValue.reference) }; + let reference = makeAbsoluteURL(rawValue.reference); + if (matched) { + dependencies.add(reference); + } + return { id: reference }; } if (!matched) { return { @@ -296,11 +347,20 @@ async function searchableLink( // getter. The store can't `toURL` a relative string, which would otherwise // degrade an expandable searchable link to `{ id }`. let resolvedRef = makeAbsoluteURL(rawValue.reference); - let loaded = await loadSearchableTarget(store, resolvedRef); - if (loaded == null) { + let result = await loadSearchableTarget(store, resolvedRef); + if (result.status === 'broken') { + // Plant the terminal sentinel on the owner's field so `getBrokenLinks` + // (which reads terminal sentinels from the data bucket) records this + // broken searchable link. This mirrors the sentinel `lazilyLoadLink` + // plants during a template render; search-doc generation plants its own + // because it drives the link load directly rather than through a template. + getDataBucket(owner).set(field.name, result.sentinel); + // A broken searchable target is still a dependency — see the sentinel + // branch above. + dependencies.add(resolvedRef); return { id: resolvedRef }; } - target = loaded; + target = result.card; } // The expanded target's data is now in the doc, so it is a dependency of the // indexed card. @@ -318,7 +378,9 @@ async function searchableLink( } // A `linksToMany` value: per-slot `{ id }` / expansion, with the absolute-URL -// id normalization `LinksToMany.queryableValue` applies. +// id normalization `LinksToMany.queryableValue` applies. A broken searchable +// slot is planted back into the backing array (which lives in the owner's data +// bucket) so `getBrokenLinks` records it — see `searchableLink`. async function searchableLinksToMany( field: Field, rawValue: any, @@ -335,12 +397,18 @@ async function searchableLinksToMany( return null; } let out: any[] = []; - for (let item of rawArrayValues(rawValue)) { + let backing = rawArrayValues(rawValue); + for (let [index, item] of backing.entries()) { if (item == null) { continue; } if (isLinkError(item) || isLinkNotFound(item)) { - out.push({ id: makeAbsoluteURL(item.reference) }); + let reference = makeAbsoluteURL(item.reference); + // A broken searchable element stays a dependency — see `searchableLink`. + if (matched) { + dependencies.add(reference); + } + out.push({ id: reference }); continue; } if (!matched) { @@ -355,12 +423,19 @@ async function searchableLinksToMany( if (isNotLoadedValue(item)) { // Resolve a relative reference before the load — see `searchableLink`. let resolvedRef = makeAbsoluteURL(item.reference); - let loaded = await loadSearchableTarget(store, resolvedRef); - if (loaded == null) { + let result = await loadSearchableTarget(store, resolvedRef); + if (result.status === 'broken') { + // Plant the sentinel into the failed slot so `getBrokenLinks` records + // this broken searchable element (see `searchableLink`). Assign through + // the proxy so the mutation reaches the data bucket the diagnostic + // reads. + rawValue[index] = result.sentinel; + // A broken searchable element stays a dependency — see `searchableLink`. + dependencies.add(resolvedRef); out.push({ id: resolvedRef }); continue; } - target = loaded; + target = result.card; } // The expanded target's data is now in the doc, so it is a dependency of // the indexed card. diff --git a/packages/boxel-cli/tests/commands/ingest-card.test.ts b/packages/boxel-cli/tests/commands/ingest-card.test.ts index 732244464d..fc08f81d8d 100644 --- a/packages/boxel-cli/tests/commands/ingest-card.test.ts +++ b/packages/boxel-cli/tests/commands/ingest-card.test.ts @@ -15,7 +15,6 @@ describe('ingest-card helpers', () => { 'bottles.0': { links: { self: '../WineBottle/a' } }, 'bottles.1': { links: { self: '../WineBottle/b' } }, label: { links: { self: '../Image/x' } }, - 'cardInfo.theme': { links: { self: null } }, }, }, }); diff --git a/packages/host/app/commands/copy-and-edit.ts b/packages/host/app/commands/copy-and-edit.ts index 083efd94d2..026636751d 100644 --- a/packages/host/app/commands/copy-and-edit.ts +++ b/packages/host/app/commands/copy-and-edit.ts @@ -150,8 +150,13 @@ export default class CopyAndEditCommand extends HostBaseCommand< let containerForFields = targetPath && this.getWrappedInstance(targetPath, parentCard); let fieldContainer = containerForFields ?? parentCard; + // Discover every declared link field, not only the ones the card has: + // this loop locates the target field to relink, and the target is + // typically empty at that point (the whole reason we're linking a copy + // into it). The `usedLinksToFieldsOnly` default would omit an unset target + // and silently skip the relink. let fields = cardApi.getFields(fieldContainer, { - usedLinksToFieldsOnly: true, + usedLinksToFieldsOnly: false, includeComputeds: false, }); @@ -278,15 +283,50 @@ export default class CopyAndEditCommand extends HostBaseCommand< card: CardAPI.CardDef, fieldName: string, ): string | undefined { - try { - let serialized = this.#cardAPI?.serializeCard(card, {}); - let relationships = (serialized?.data as any)?.relationships ?? {}; - return Object.keys(relationships).find( - (key) => key === fieldName || key.endsWith(`.${fieldName}`), - ); - } catch { + let api = this.#cardAPI; + if (!api) { return undefined; } + // Resolve a leaf field name to its full dotted path (e.g. `theme` -> + // `cardInfo.theme`) by walking the declared field structure, not a + // serialization: the target we're linking into is typically unset, so it + // may not appear in the card's serialized relationships at all. + let search = ( + owner: CardAPI.BaseDef | typeof CardAPI.BaseDef, + prefix: string, + ): string | undefined => { + let fields: Record; + try { + fields = api.getFields(owner as any, { includeComputeds: false }); + } catch { + return undefined; + } + for (let [name, field] of Object.entries(fields)) { + if (!field) { + continue; + } + let path = prefix ? `${prefix}.${name}` : name; + if ( + name === fieldName && + (field.fieldType === 'linksTo' || field.fieldType === 'linksToMany') + ) { + return path; + } + if ( + (field.fieldType === 'contains' || + field.fieldType === 'containsMany') && + 'card' in field && + field.card + ) { + let nested = search(field.card, path); + if (nested) { + return nested; + } + } + } + return undefined; + }; + return search(card, ''); } // Example: dotGetter('cardInfo.theme', card) -> card.cardInfo.theme diff --git a/packages/host/app/components/card-prerender.gts b/packages/host/app/components/card-prerender.gts index 976c5b3aaa..70c5e02068 100644 --- a/packages/host/app/components/card-prerender.gts +++ b/packages/host/app/components/card-prerender.gts @@ -32,6 +32,7 @@ import { VISIT_PASS_ORDER, serializeRenderRouteOptions, cleanCapturedHTML, + snapshotRuntimeDependencies, } from '@cardstack/runtime-common'; import { readFileAsText as _readFileAsText } from '@cardstack/runtime-common/stream'; @@ -202,13 +203,19 @@ export default class CardPrerender extends Component { // Composite visit task — runs the caller-selected subset of // {fileExtract, cardRender, fileRender} on a shared nonce and with a single - // clearCache consumption. Mirrors the server-side prerenderVisitAttempt. + // clearCache consumption. Mirrors the server-side prerenderVisitAttempt, + // including its visitType bifurcation: an 'index' visit runs the extract, + // the card's meta + icon and the file's icon — never an html-format + // render; a 'prerender-html' visit runs only the html formats + markdown. + // No visitType runs the fused union. private prerenderVisitTask = enqueueTask( async ({ url, + visitType, renderOptions, fileData, types, + cardTypes, }: PrerenderVisitArgs): Promise => { this.#nonce++; // Clear any residual render error from a previous visit so the earliest @@ -219,8 +226,12 @@ export default class CardPrerender extends Component { Boolean(renderOptions?.clearCache), ); let baseOptions: RenderRouteOptions = { ...(renderOptions ?? {}) }; + let runIndexSteps = visitType !== 'prerender-html'; + let runHtmlSteps = visitType !== 'index'; let requested = { - fileExtract: Boolean(baseOptions.fileExtract), + // The extract belongs to the index half; a prerender-html visit + // never runs it even when the flag rides along on shared options. + fileExtract: Boolean(baseOptions.fileExtract) && runIndexSteps, cardRender: Boolean(baseOptions.cardRender), fileRender: Boolean(baseOptions.fileRender), }; @@ -333,6 +344,7 @@ export default class CardPrerender extends Component { let embeddedHTML: Record | null = null; let fittedHTML: Record | null = null; let markdown: string | null = null; + let capturedDeps: string[] | null = null; let meta: PrerenderMeta = { serialized: null, searchDoc: null, @@ -344,49 +356,83 @@ export default class CardPrerender extends Component { await this.#primeCardType(url, context); let subsequentRenderOptions = omitOneTimeOptions(initialRenderOptions); - isolatedHTML = await this.renderHTML.perform( - url, - 'isolated', - 0, - initialRenderOptions, - ); - meta = await this.renderMeta.perform(url, subsequentRenderOptions); - headHTML = await this.renderHTML.perform( - url, - 'head', - 0, - subsequentRenderOptions, - ); - atomHTML = await this.renderHTML.perform( - url, - 'atom', - 0, - subsequentRenderOptions, - ); - iconHTML = await this.renderIcon.perform( - url, - subsequentRenderOptions, - ); - markdown = await this.renderHTML.perform( - url, - 'markdown', - 0, - subsequentRenderOptions, - ); - if (meta?.types) { - embeddedHTML = await this.renderAncestors.perform( + if (runHtmlSteps) { + isolatedHTML = await this.renderHTML.perform( + url, + 'isolated', + 0, + initialRenderOptions, + ); + if (visitType === 'prerender-html') { + // The card's runtime deps normally ride on render.meta, which + // a prerender-html visit never runs. Snapshot the tracking + // session directly after the isolated render settles so the + // HTML rendering still reports what it pulled in — the + // indexing job unions this with the index visit's meta deps. + capturedDeps = this.network.virtualNetwork.unresolveURLs( + snapshotRuntimeDependencies({ excludeQueryOnly: true }).deps, + ); + } + } + if (runIndexSteps) { + meta = await this.renderMeta.perform( + url, + runHtmlSteps ? subsequentRenderOptions : initialRenderOptions, + ); + } + if (runHtmlSteps) { + headHTML = await this.renderHTML.perform( url, - 'embedded', - meta.types, + 'head', + 0, + subsequentRenderOptions, + ); + atomHTML = await this.renderHTML.perform( + url, + 'atom', + 0, subsequentRenderOptions, ); - fittedHTML = await this.renderAncestors.perform( + } + if (runIndexSteps) { + iconHTML = await this.renderIcon.perform( url, - 'fitted', - meta.types, subsequentRenderOptions, ); } + if (runHtmlSteps) { + markdown = await this.renderHTML.perform( + url, + 'markdown', + 0, + subsequentRenderOptions, + ); + // The ancestor type chain drives the fitted/embedded renders. A + // prerender-html visit takes the chain the caller passed (the + // indexing job forwards the index visit's types) and resolves it + // itself via render.meta only as a fallback; the fused visit + // already holds it from the meta step above. + let typesForAncestors = runIndexSteps + ? meta?.types + : cardTypes?.length + ? cardTypes + : (await this.renderMeta.perform(url, subsequentRenderOptions)) + ?.types; + if (typesForAncestors) { + embeddedHTML = await this.renderAncestors.perform( + url, + 'embedded', + typesForAncestors, + subsequentRenderOptions, + ); + fittedHTML = await this.renderAncestors.perform( + url, + 'fitted', + typesForAncestors, + subsequentRenderOptions, + ); + } + } } catch (e: any) { try { cardError = { ...JSON.parse(e.message), type: 'instance-error' }; @@ -425,6 +471,7 @@ export default class CardPrerender extends Component { } response.card = { ...meta, + ...(capturedDeps ? { deps: capturedDeps } : {}), isolatedHTML, headHTML, atomHTML, @@ -490,48 +537,56 @@ export default class CardPrerender extends Component { try { let subsequentRenderOptions = omitOneTimeOptions(initialRenderOptions); - isolatedHTML = await this.renderHTML.perform( - url, - 'isolated', - 0, - initialRenderOptions, - ); - headHTML = await this.renderHTML.perform( - url, - 'head', - 0, - subsequentRenderOptions, - ); - atomHTML = await this.renderHTML.perform( - url, - 'atom', - 0, - subsequentRenderOptions, - ); - iconHTML = await this.renderIcon.perform( - url, - subsequentRenderOptions, - ); - markdown = await this.renderHTML.perform( - url, - 'markdown', - 0, - subsequentRenderOptions, - ); - if (effectiveTypes?.length) { - embeddedHTML = await this.renderAncestors.perform( + if (runHtmlSteps) { + isolatedHTML = await this.renderHTML.perform( url, - 'embedded', - effectiveTypes, + 'isolated', + 0, + initialRenderOptions, + ); + headHTML = await this.renderHTML.perform( + url, + 'head', + 0, subsequentRenderOptions, ); - fittedHTML = await this.renderAncestors.perform( + atomHTML = await this.renderHTML.perform( url, - 'fitted', - effectiveTypes, + 'atom', + 0, subsequentRenderOptions, ); } + if (runIndexSteps) { + // The file's icon belongs to the index half; in an index visit + // it is the pass's only render. + iconHTML = await this.renderIcon.perform( + url, + runHtmlSteps ? subsequentRenderOptions : initialRenderOptions, + ); + } + if (runHtmlSteps) { + markdown = await this.renderHTML.perform( + url, + 'markdown', + 0, + subsequentRenderOptions, + ); + if (effectiveTypes?.length) { + embeddedHTML = await this.renderAncestors.perform( + url, + 'embedded', + effectiveTypes, + subsequentRenderOptions, + ); + fittedHTML = await this.renderAncestors.perform( + url, + 'fitted', + effectiveTypes, + subsequentRenderOptions, + ); + } + } } catch (e: any) { try { fileError = { ...JSON.parse(e.message), type: 'file-error' }; diff --git a/packages/host/app/routes/render.ts b/packages/host/app/routes/render.ts index cadf4eb1f2..9f6a6e1061 100644 --- a/packages/host/app/routes/render.ts +++ b/packages/host/app/routes/render.ts @@ -167,6 +167,7 @@ export default class RenderRoute extends Route { // invalidation is handled by `fetchSearchDoc`'s entry-time // jobId-change clear (and by `resetState` on harder resets). (globalThis as any).__renderModel = undefined; + (globalThis as any).__boxelRenderCapturedDeps = undefined; (globalThis as any).__docsInFlight = undefined; (globalThis as any).__boxelRenderStage = undefined; // CS-10872: also tear down the stage setter + timestamp. Without @@ -349,6 +350,9 @@ export default class RenderRoute extends Route { { id, nonce }: { id: string; nonce: string }, parsedOptions: ReturnType, ): Promise { + // Reset before any await so a reader can never see a previous card's + // settle-time snapshot; #settleModelAfterRender repopulates it. + (globalThis as any).__boxelRenderCapturedDeps = undefined; if (parsedOptions.clearCache) { this.loaderService.resetLoader({ clearFetchCache: true, @@ -646,6 +650,14 @@ export default class RenderRoute extends Route { model.capturedDeps = snapshotRuntimeDependencies({ excludeQueryOnly: true, }).deps; + // Publish the settle-time dependency snapshot (unresolved/prefix form, + // matching what render.meta reports as `deps`) for visits that never run + // the meta route: a prerender-html visit reads this global after its + // isolated render so the HTML rendering still reports the dependencies + // it pulled in. Cleared at the top of #buildModel so a later card in the + // same tab can never read a predecessor's snapshot. + (globalThis as any).__boxelRenderCapturedDeps = + this.network.virtualNetwork.unresolveURLs(model.capturedDeps); renderReadyLogger.debug( `settleModelAfterRender done cardId=${model.cardId} deps=${model.capturedDeps?.length ?? 0}`, ); @@ -810,6 +822,7 @@ export default class RenderRoute extends Route { // (and the entire ApplicationInstance) on globalThis. (globalThis as any).__boxelRenderContext = undefined; (globalThis as any).__renderModel = undefined; + (globalThis as any).__boxelRenderCapturedDeps = undefined; (globalThis as any).__docsInFlight = undefined; (globalThis as any).__waitForRenderLoadStability = undefined; }); diff --git a/packages/host/app/routes/render/meta.ts b/packages/host/app/routes/render/meta.ts index 39f9167480..47cdbec211 100644 --- a/packages/host/app/routes/render/meta.ts +++ b/packages/host/app/routes/render/meta.ts @@ -23,6 +23,7 @@ import { import type CardService from '@cardstack/host/services/card-service'; import type NetworkService from '@cardstack/host/services/network'; +import type RenderStoreService from '@cardstack/host/services/render-store'; import type { BaseDef, @@ -38,9 +39,24 @@ export type Model = PrerenderMeta | RenderError | undefined; const computePerfLog = logger('host:computed-perf'); +// Bounds for the searchable-driven load-settle loop below. Match the +// template-render settle in the /render route (READY_SETTLE_MAX_PASSES / +// READY_SETTLE_REQUIRED_STABLE_PASSES): keep pulling + waiting until the +// store's load generation holds steady for a couple of passes, capped so a +// pathological graph can't loop forever. +const SEARCHABLE_SETTLE_MAX_PASSES = 20; +const SEARCHABLE_SETTLE_REQUIRED_STABLE_PASSES = 2; + +// The base module whose generator produces every search doc. Recorded as a +// dependency of the meta output directly (see the deps union below) rather +// than relying on its per-loader-cached module-load hook to fire during a +// given render. `unresolveURLs` maps it to `@cardstack/base/searchable`. +const SEARCHABLE_MODULE_URL = 'https://cardstack.com/base/searchable'; + export default class RenderMetaRoute extends Route { @service declare cardService: CardService; @service declare private network: NetworkService; + @service('render-store') declare private store: RenderStoreService; async model(_: unknown, transition: Transition) { let api = await this.cardService.getAPI(); @@ -59,16 +75,53 @@ export default class RenderMetaRoute extends Route { return; } - let deps = - renderModel?.capturedDeps ?? - snapshotRuntimeDependencies({ excludeQueryOnly: true }).deps; - - // The search doc comes from the searchable-driven generator, which lives in - // its own base module (off card-api, so it stays out of every card's - // dependency closure). It derives link depth from the explicit `searchable` - // annotations rather than from what the render happened to load. + // The search doc comes from the searchable-driven generator in its own base + // module. It derives link depth from the explicit `searchable` annotations + // rather than from what the render happened to load. let searchable = await this.cardService.getSearchable(); + // Drive the instance's linked-field loading to quiescence before + // serializing. The searchable annotations name which links to pull, and + // the card's own contained/computed fields read links too — both fire + // lazy loads through the field getter (tracked on the store). Nothing + // awaits those loads inline (a computed reading a not-yet-loaded link + // sees `undefined`; a broken link's terminal sentinel isn't planted + // until its load settles), so pull-then-wait here, repeating until the + // store's load generation holds steady: each pass loads whatever the + // current state newly permits — a deeper searchable hop once its parent + // resolved, a computed's link once the computed re-reads — and + // `store.loaded()` drains it. Cycles clip via the generator's own stack + // guard, so a ring loads its nodes once and then quiesces. + // + // This reproduces, for the search doc, the settle the /render route + // provides for template renders (its readyPromise waits on the same + // `store.loaded()` after the template pulls the links). The two are + // complementary: when a template render already loaded the graph (a fused + // visit, or an HTML render sharing this tab), the first pass finds every + // target resident and `store.loaded()` resolves immediately — there is + // nothing left to wait for. + await this.#settleSearchableLoads(instance, searchable); + + // Union the render route's captured deps with a fresh snapshot: the + // settle loop above loaded links through the tracked getter (each a + // recorded dependency), and those loads land in the still-open tracking + // session but after `capturedDeps` was snapshotted. A render that never + // pulled a link (an index visit with no HTML render) would otherwise + // drop the edges searchable just followed. + // + // The searchable module itself is loaded through a per-loader cache, so its + // module-load hook only fires (and only records a dependency) on the first + // pull in a given render tab — a warm tab would omit it. Building the search + // doc always consumes it, so record it unconditionally, independent of that + // load-hook timing. + let deps = [ + ...new Set([ + SEARCHABLE_MODULE_URL, + ...(renderModel?.capturedDeps ?? []), + ...snapshotRuntimeDependencies({ excludeQueryOnly: true }).deps, + ]), + ]; + // Open a synchronous compute-memo pass over serializeCard. Computed fields // invoked through the descriptor or through peekAtField hit the per-instance // memo instead of re-running `computeVia` — one compute per distinct @@ -114,12 +167,19 @@ export default class RenderMetaRoute extends Route { ), }) as SingleCardDocument; serializeMs = performance.now() - serializeStart; + // Emulate the on-disk file serialization: a card file holds only the + // card's own resource — relationship slots keep their `links` but drop + // the resolved `data`, and no linked neighbors ride along in `included`. + // The searchable settle above may have loaded link targets into the + // store, and `serializeCard` walks whatever is resident into `included`; + // strip both so the serialized instance is a pure function of the card's + // own data, independent of which targets happen to be loaded. for (let { relationship } of relationshipEntries( serialized.data.relationships, )) { - // we want to emulate the file serialization here delete relationship.data; } + delete serialized.included; } finally { if (passOpen && typeof api.endComputePass === 'function') { passSnapshot = api.endComputePass(); @@ -199,6 +259,50 @@ export default class RenderMetaRoute extends Route { diagnostics, }; } + + // Pull the instance's searchable-path links (and the links its + // contained/computed fields read) and wait for the store to settle, + // repeating until the load generation is stable. Running the searchable + // generator IS the pull — it reads every field the search doc and pristine + // doc will read, firing each link's lazy load through the tracked getter — + // and `store.loaded()` is the wait. The intermediate search docs are + // discarded (a throwaway dependency set); the authoritative generation runs + // afterward against the settled store. See the call site for why this is + // needed and how it composes with the /render template settle. + async #settleSearchableLoads( + instance: CardDef, + searchable: Awaited>, + ): Promise { + let observedGeneration = this.store.loadGeneration; + let stablePasses = 0; + for (let pass = 0; pass < SEARCHABLE_SETTLE_MAX_PASSES; pass++) { + // A computed that reads a not-yet-loaded link (e.g. + // `this.author.firstName`) throws on the first pass — the getter fires + // the lazy load, then the read of the still-`undefined` target throws. + // The load it fired is what we wait on next, so swallow the throw and + // let it settle: a later pass re-runs the computed against the loaded + // target. A genuine (non-load-race) failure resurfaces in the + // authoritative generation the caller runs after this settles. + try { + await searchable.searchDocFromFields(instance); + } catch { + // intentionally ignored during settle — see above + } + await this.store.loaded(); + let nextGeneration = this.store.loadGeneration; + if (nextGeneration === observedGeneration) { + if (++stablePasses >= SEARCHABLE_SETTLE_REQUIRED_STABLE_PASSES) { + return; + } + } else { + observedGeneration = nextGeneration; + stablePasses = 0; + } + } + computePerfLog.warn( + `render.meta searchable settle for ${instance.id} did not reach a stable load generation within ${SEARCHABLE_SETTLE_MAX_PASSES} passes; proceeding with the current store state`, + ); + } } export function getClass(instance: CardDef): typeof CardDef { diff --git a/packages/host/tests/acceptance/code-submode/create-file-test.gts b/packages/host/tests/acceptance/code-submode/create-file-test.gts index fc915979f9..a6d96e81ff 100644 --- a/packages/host/tests/acceptance/code-submode/create-file-test.gts +++ b/packages/host/tests/acceptance/code-submode/create-file-test.gts @@ -605,18 +605,7 @@ module('Acceptance | code submode | create-file tests', function (hooks) { ); assert.deepEqual( json.data.relationships, - { - pet: { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, + undefined, 'relationships data is correct', ); deferred.fulfill(); diff --git a/packages/host/tests/acceptance/code-submode/editor-test.ts b/packages/host/tests/acceptance/code-submode/editor-test.ts index 81258872cd..5892893ebc 100644 --- a/packages/host/tests/acceptance/code-submode/editor-test.ts +++ b/packages/host/tests/acceptance/code-submode/editor-test.ts @@ -609,13 +609,6 @@ module('Acceptance | code submode | editor tests', function (hooks) { notes: null, }, }, - relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri(`../pet`), diff --git a/packages/host/tests/acceptance/file-def-test.gts b/packages/host/tests/acceptance/file-def-test.gts index 480a1c7d9f..439e38a143 100644 --- a/packages/host/tests/acceptance/file-def-test.gts +++ b/packages/host/tests/acceptance/file-def-test.gts @@ -156,11 +156,6 @@ module('Acceptance | file def', function (hooks) { id: './env-indexing-operations.md', }, }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { diff --git a/packages/host/tests/helpers/index.gts b/packages/host/tests/helpers/index.gts index ad1496f45f..5bacbc14bc 100644 --- a/packages/host/tests/helpers/index.gts +++ b/packages/host/tests/helpers/index.gts @@ -1026,13 +1026,6 @@ export const SYSTEM_CARD_FIXTURE_CONTENTS: RealmContents = { toolsSupported: true, reasoningEffort: 'minimal', }, - relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: '@cardstack/base/system-card', @@ -1055,13 +1048,6 @@ export const SYSTEM_CARD_FIXTURE_CONTENTS: RealmContents = { modelId: 'anthropic/claude-sonnet-4.6', toolsSupported: true, }, - relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: '@cardstack/base/system-card', @@ -1084,13 +1070,6 @@ export const SYSTEM_CARD_FIXTURE_CONTENTS: RealmContents = { modelId: 'anthropic/claude-sonnet-4.5', toolsSupported: true, }, - relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: '@cardstack/base/system-card', @@ -1113,13 +1092,6 @@ export const SYSTEM_CARD_FIXTURE_CONTENTS: RealmContents = { modelId: 'anthropic/claude-3.7-sonnet', toolsSupported: true, }, - relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: '@cardstack/base/system-card', diff --git a/packages/host/tests/integration/commands/patch-instance-test.gts b/packages/host/tests/integration/commands/patch-instance-test.gts index d0af1654a3..acc0ff769f 100644 --- a/packages/host/tests/integration/commands/patch-instance-test.gts +++ b/packages/host/tests/integration/commands/patch-instance-test.gts @@ -142,19 +142,7 @@ module('Integration | commands | patch-instance', function (hooks) { ); assert.deepEqual( instance.relationships, - { - bestFriend: { - links: { - self: null, - }, - }, - friends: { - links: { - self: null, - }, - }, - 'cardInfo.theme': { links: { self: null } }, - }, + undefined, 'the relationships are correct', ); }); @@ -212,19 +200,7 @@ module('Integration | commands | patch-instance', function (hooks) { ); assert.deepEqual( instance.relationships, - { - bestFriend: { - links: { - self: null, - }, - }, - friends: { - links: { - self: null, - }, - }, - 'cardInfo.theme': { links: { self: null } }, - }, + undefined, 'the relationships are correct', ); }); @@ -400,12 +376,6 @@ module('Integration | commands | patch-instance', function (hooks) { self: `./jade`, }, }, - friends: { - links: { - self: null, - }, - }, - 'cardInfo.theme': { links: { self: null } }, }, 'the relationships are correct', ); @@ -466,14 +436,8 @@ module('Integration | commands | patch-instance', function (hooks) { assert.deepEqual( instance.relationships, { - bestFriend: { - links: { - self: null, - }, - }, 'friends.0': { links: { self: `./germaine` } }, 'friends.1': { links: { self: `./queenzy` } }, - 'cardInfo.theme': { links: { self: null } }, }, 'the relationships are correct', ); @@ -619,13 +583,6 @@ module('Integration | commands | patch-instance', function (hooks) { instance.relationships, { bestFriend: { links: { self: `./queenzy` } }, - 'cardInfo.cardThumbnail': { links: { self: null } }, - 'cardInfo.theme': { links: { self: null } }, - friends: { - links: { - self: null, - }, - }, }, 'the relationships are correct', ); diff --git a/packages/host/tests/integration/components/card-copy-test.gts b/packages/host/tests/integration/components/card-copy-test.gts index bca911d5d5..68795a6712 100644 --- a/packages/host/tests/integration/components/card-copy-test.gts +++ b/packages/host/tests/integration/components/card-copy-test.gts @@ -1020,7 +1020,6 @@ module('Integration | card-copy', function (hooks) { id: `${testRealmURL}Pet/mango`, }, }, - 'cardInfo.theme': { links: { self: null } }, }); assert.strictEqual(json.included?.length, 1); // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain @@ -1155,7 +1154,6 @@ module('Integration | card-copy', function (hooks) { id: `${testRealm2URL}Pet/paper`, }, }, - 'cardInfo.theme': { links: { self: null } }, }); assert.strictEqual(json.included?.length, 1); // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain diff --git a/packages/host/tests/integration/components/operator-mode-links-test.gts b/packages/host/tests/integration/components/operator-mode-links-test.gts index 70908d3afe..e4afad965a 100644 --- a/packages/host/tests/integration/components/operator-mode-links-test.gts +++ b/packages/host/tests/integration/components/operator-mode-links-test.gts @@ -511,18 +511,6 @@ module('Integration | operator-mode | links', function (hooks) { notes: null, }, }, - relationships: { - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: '../pet', diff --git a/packages/host/tests/integration/components/serialization-test.gts b/packages/host/tests/integration/components/serialization-test.gts index dbeb128a20..99b9fdc99b 100644 --- a/packages/host/tests/integration/components/serialization-test.gts +++ b/packages/host/tests/integration/components/serialization-test.gts @@ -923,8 +923,9 @@ module('Integration | serialization', function (hooks) { let ref = { module: `${testModuleRealm}person`, name: 'Person' }; let driver = new DriverCard({ ref }); - let serializedRef = serializeCard(driver, { includeUnrenderedFields: true }) - .data.attributes?.ref; + let serializedRef = serializeCard(driver, { + includeUnrenderedFields: true, + }).data.attributes?.ref; assert.notStrictEqual( serializedRef, ref, @@ -1081,7 +1082,9 @@ module('Integration | serialization', function (hooks) { ); await saveCard(mango, `${testRealmURL}Pet/mango`, loader); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -1214,7 +1217,9 @@ module('Integration | serialization', function (hooks) { pet: mango, }); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -1570,7 +1575,9 @@ module('Integration | serialization', function (hooks) { let hassan = new Person({ firstName: 'Hassan' }); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -1818,7 +1825,9 @@ module('Integration | serialization', function (hooks) { let mango = new Person({ firstName: 'Mango' }); let hassan = new Person({ firstName: 'Hassan', friend: mango }); await saveCard(mango, `${testRealmURL}Person/mango`, loader); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -2052,7 +2061,9 @@ module('Integration | serialization', function (hooks) { `${testRealmURL}Toy/spookyToiletPaper`, loader, ); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -2186,7 +2197,9 @@ module('Integration | serialization', function (hooks) { `${testRealmURL}Toy/spookyToiletPaper`, loader, ); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -2324,7 +2337,9 @@ module('Integration | serialization', function (hooks) { `${testRealmURL}Toy/spookyToiletPaper`, loader, ); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -3086,13 +3101,6 @@ module('Integration | serialization', function (hooks) { }, cardInfo, }, - relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: testRRI('test-cards'), @@ -3851,7 +3859,9 @@ module('Integration | serialization', function (hooks) { cardDescription: 'Introductory post', cardThumbnailURL: './intro.png', }); - let payload = serializeCard(firstPost, { includeUnrenderedFields: true }); + let payload = serializeCard(firstPost, { + includeUnrenderedFields: true, + }); assert.deepEqual( payload, { @@ -3924,7 +3934,9 @@ module('Integration | serialization', function (hooks) { cardDescription: 'A dog', }), }); - let payload = serializeCard(firstPost, { includeUnrenderedFields: true }); + let payload = serializeCard(firstPost, { + includeUnrenderedFields: true, + }); assert.deepEqual(payload, { data: { lid: firstPost[localId], @@ -3996,7 +4008,9 @@ module('Integration | serialization', function (hooks) { department: 'wagging', }), }); - let payload = serializeCard(firstPost, { includeUnrenderedFields: true }); + let payload = serializeCard(firstPost, { + includeUnrenderedFields: true, + }); assert.deepEqual(payload, { data: { lid: firstPost[localId], @@ -4101,7 +4115,9 @@ module('Integration | serialization', function (hooks) { }), }), }); - let payload = serializeCard(firstPost, { includeUnrenderedFields: true }); + let payload = serializeCard(firstPost, { + includeUnrenderedFields: true, + }); assert.deepEqual(payload, { data: { lid: firstPost[localId], @@ -4507,7 +4523,9 @@ module('Integration | serialization', function (hooks) { }), }); - let serialized = serializeCard(article, { includeUnrenderedFields: true }); + let serialized = serializeCard(article, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { @@ -5672,13 +5690,6 @@ module('Integration | serialization', function (hooks) { birthdate: '2019-10-30', cardInfo, }, - relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: testRRI('test-cards'), @@ -5705,11 +5716,6 @@ module('Integration | serialization', function (hooks) { cardInfo, }, relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, cardTheme: { links: { self: null, @@ -6588,7 +6594,9 @@ module('Integration | serialization', function (hooks) { let hassan = new Person({ firstName: 'Hassan' }); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -6813,7 +6821,9 @@ module('Integration | serialization', function (hooks) { } } - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { lid: hassan[localId], @@ -6894,7 +6904,9 @@ module('Integration | serialization', function (hooks) { await saveCard(mango, `${testRealmURL}Person/mango`, loader); await saveCard(vanGogh, `${testRealmURL}Person/vanGogh`, loader); await saveCard(hassan, `${testRealmURL}Person/hassan`, loader); - let serialized = serializeCard(hassan, { includeUnrenderedFields: true }); + let serialized = serializeCard(hassan, { + includeUnrenderedFields: true, + }); assert.deepEqual(serialized, { data: { type: 'card', diff --git a/packages/host/tests/integration/realm-indexing-test.gts b/packages/host/tests/integration/realm-indexing-test.gts index eb2182f8e1..92159debb7 100644 --- a/packages/host/tests/integration/realm-indexing-test.gts +++ b/packages/host/tests/integration/realm-indexing-test.gts @@ -160,9 +160,6 @@ module(`Integration | realm indexing`, function (hooks) { cardDescription: null, cardThumbnailURL: null, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: rri('@cardstack/base/card-api'), @@ -437,7 +434,6 @@ module(`Integration | realm indexing`, function (hooks) { self: `../Person/owner`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -529,14 +525,6 @@ module(`Integration | realm indexing`, function (hooks) { cardThumbnailURL: null, cardInfo, }, - relationships: { - owner: { - links: { - self: null, - }, - }, - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: rri(`${testModuleRealm}pet`), @@ -585,14 +573,6 @@ module(`Integration | realm indexing`, function (hooks) { cardThumbnailURL: null, cardInfo, }, - relationships: { - owner: { - links: { - self: null, - }, - }, - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: rri(`${testModuleRealm}pet`), @@ -904,7 +884,6 @@ module(`Integration | realm indexing`, function (hooks) { self: `../Person/owner`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -1000,7 +979,6 @@ module(`Integration | realm indexing`, function (hooks) { self: `../Person/owner`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -1091,10 +1069,6 @@ module(`Integration | realm indexing`, function (hooks) { containedExamples: [], cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - linkedExamples: { links: { self: null } }, - }, meta: { adoptsFrom: { module: rri('@cardstack/base/spec'), @@ -1221,10 +1195,6 @@ module(`Integration | realm indexing`, function (hooks) { containedExamples: [], cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - linkedExamples: { links: { self: null } }, - }, meta: { adoptsFrom: { module: rri('@cardstack/base/spec'), @@ -1299,9 +1269,6 @@ module(`Integration | realm indexing`, function (hooks) { cardTitle: null, cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: skillCardRef, lastModified: adapter.lastModifiedMap.get( @@ -1425,9 +1392,20 @@ module(`Integration | realm indexing`, function (hooks) { entry.error.errorDetail.message, 'Encountered error rendering HTML for card: intentional error', ); - assert.deepEqual(entry.error.errorDetail.deps, [ - `${testRealmURL}boom`, - ]); + // An errored card records its full render closure as deps, not just + // its own module: any change to a transitive dependency could be the + // one that fixes the render, so the errored card must reindex when + // any of them changes. Assert the meaningful members rather than the + // whole (large, churn-prone) closure. + let errorDeps = entry.error.errorDetail.deps ?? []; + assert.ok( + errorDeps.includes(`${testRealmURL}boom`), + "error deps include the card's own module", + ); + assert.ok( + errorDeps.includes(`${testRealmURL}@cardstack/base/card-api`), + 'error deps include transitive render dependencies (the full closure)', + ); } else { assert.ok('false', 'expected search entry to be an error document'); } @@ -1819,13 +1797,6 @@ module(`Integration | realm indexing`, function (hooks) { cardThumbnailURL: null, cardTitle: 'Untitled Card', }, - relationships: { - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri(`./person`), @@ -2367,7 +2338,6 @@ module(`Integration | realm indexing`, function (hooks) { 'appointment.contact.pet': { links: { self: `./mango` }, }, - 'cardInfo.theme': { links: { self: null } }, }); } else { assert.ok( @@ -2506,7 +2476,6 @@ module(`Integration | realm indexing`, function (hooks) { self: `../Chain/2`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -2549,9 +2518,6 @@ module(`Integration | realm indexing`, function (hooks) { cardThumbnailURL: `Ethereum Mainnet-icon.png`, cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: rri(`${testModuleRealm}chain`), @@ -2589,9 +2555,6 @@ module(`Integration | realm indexing`, function (hooks) { cardThumbnailURL: `Polygon-icon.png`, cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: rri(`${testModuleRealm}chain`), @@ -3047,11 +3010,6 @@ module(`Integration | realm indexing`, function (hooks) { cardInfo, }, relationships: { - friend: { - links: { - self: null, - }, - }, 'pets.0': { links: { self: `../Pet/mango` }, data: { id: `${testRealmURL}Pet/mango`, type: 'card' }, @@ -3060,7 +3018,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: `../Pet/vanGogh` }, data: { id: `${testRealmURL}Pet/vanGogh`, type: 'card' }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -3100,10 +3057,6 @@ module(`Integration | realm indexing`, function (hooks) { cardThumbnailURL: null, cardInfo, }, - relationships: { - owner: { links: { self: null } }, - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: testModuleRRI('pet'), @@ -3138,10 +3091,6 @@ module(`Integration | realm indexing`, function (hooks) { cardThumbnailURL: null, cardInfo, }, - relationships: { - owner: { links: { self: null } }, - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: testModuleRRI('pet'), @@ -3225,10 +3174,6 @@ module(`Integration | realm indexing`, function (hooks) { 'PetPerson/burcu.json': { data: { attributes: { firstName: 'Burcu' }, - relationships: { - pets: { links: { self: null } }, - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}pet-person`, @@ -3261,11 +3206,6 @@ module(`Integration | realm indexing`, function (hooks) { cardThumbnailURL: null, cardInfo, }, - relationships: { - pets: { links: { self: null } }, - friend: { links: { self: null } }, - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: testModuleRRI('pet-person'), @@ -3402,10 +3342,6 @@ module(`Integration | realm indexing`, function (hooks) { isField: false, cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - linkedExamples: { links: { self: null } }, - }, meta: { adoptsFrom: { module: rri('@cardstack/base/spec'), @@ -3528,13 +3464,6 @@ module(`Integration | realm indexing`, function (hooks) { cardDescription: 'Dog friend', cardThumbnailURL: 'van-gogh.jpg', }, - relationships: { - friend: { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}friend`, @@ -3569,7 +3498,6 @@ module(`Integration | realm indexing`, function (hooks) { self: `./mango`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -3721,7 +3649,6 @@ module(`Integration | realm indexing`, function (hooks) { id: `${testRealmURL}Friend/mango`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -3771,7 +3698,6 @@ module(`Integration | realm indexing`, function (hooks) { id: `${testRealmURL}Friend/hassan`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -3869,7 +3795,6 @@ module(`Integration | realm indexing`, function (hooks) { id: `${testRealmURL}Friend/hassan`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -3919,7 +3844,6 @@ module(`Integration | realm indexing`, function (hooks) { id: `${testRealmURL}Friend/mango`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -4047,7 +3971,6 @@ module(`Integration | realm indexing`, function (hooks) { id: `${testRealmURL}Friend/hassan`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -4201,7 +4124,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './vanGogh' }, data: { type: 'card', id: vanGoghID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4246,7 +4168,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './hassan' }, data: { type: 'card', id: hassanID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4285,7 +4206,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './hassan' }, data: { type: 'card', id: hassanID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4377,7 +4297,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './hassan' }, data: { type: 'card', id: hassanID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4425,7 +4344,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './vanGogh' }, data: { type: 'card', id: vanGoghID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4464,7 +4382,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './hassan' }, data: { type: 'card', id: hassanID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4562,7 +4479,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './hassan' }, data: { type: 'card', id: hassanID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4610,7 +4526,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './vanGogh' }, data: { type: 'card', id: vanGoghID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4649,7 +4564,6 @@ module(`Integration | realm indexing`, function (hooks) { links: { self: './hassan' }, data: { type: 'card', id: hassanID }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: friendsRef, @@ -4793,6 +4707,7 @@ module(`Integration | realm indexing`, function (hooks) { '@cardstack/base/number/components/stat', '@cardstack/base/number/util/index', '@cardstack/base/query-field-support', + '@cardstack/base/searchable', '@cardstack/base/shared-state', '@cardstack/base/string', '@cardstack/base/text-input-validator', @@ -4958,6 +4873,7 @@ module(`Integration | realm indexing`, function (hooks) { '@cardstack/base/number/components/stat', '@cardstack/base/number/util/index', '@cardstack/base/query-field-support', + '@cardstack/base/searchable', '@cardstack/base/shared-state', '@cardstack/base/spec', '@cardstack/base/string', diff --git a/packages/host/tests/integration/realm-test.gts b/packages/host/tests/integration/realm-test.gts index 0b8d48a1db..4835282a78 100644 --- a/packages/host/tests/integration/realm-test.gts +++ b/packages/host/tests/integration/realm-test.gts @@ -118,9 +118,6 @@ module('Integration | realm', function (hooks) { cardThumbnailURL: null, cardTitle: 'Untitled Card', }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: '@cardstack/base/card-api', @@ -155,9 +152,6 @@ module('Integration | realm', function (hooks) { cardDescription: null, cardThumbnailURL: null, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}person`, @@ -181,7 +175,6 @@ module('Integration | realm', function (hooks) { self: `${testRealmURL}dir/owner`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -227,7 +220,6 @@ module('Integration | realm', function (hooks) { id: `${testRealmURL}dir/owner`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -260,9 +252,6 @@ module('Integration | realm', function (hooks) { fullName: 'Hassan Abdel-Rahman', cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}person`, @@ -343,11 +332,6 @@ module('Integration | realm', function (hooks) { `${testModuleRealm}hassan`, 'owner data id points to other realm', ); - assert.strictEqual( - json.data.relationships['cardInfo.theme'].links.self, - null, - 'mango theme is null', - ); assert.strictEqual( json.data.meta.lastModified, adapter.lastModifiedMap.get(`${testRealmURL}dir/mango.json`), @@ -366,11 +350,6 @@ module('Integration | realm', function (hooks) { `${testModuleRealm}hassan`, 'included hassan id', ); - assert.strictEqual( - hassan.relationships['cardInfo.theme'].links.self, - null, - 'included hassan theme is null', - ); }); test("realm can route requests correctly when mounted in the origin's subdir", async function (assert) { @@ -705,7 +684,6 @@ module('Integration | realm', function (hooks) { id: `${testRealmURL}dir/owner`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -738,9 +716,6 @@ module('Integration | realm', function (hooks) { fullName: 'Hassan Abdel-Rahman', cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}person`, @@ -947,9 +922,6 @@ module('Integration | realm', function (hooks) { lastName: 'Abdel-Rahman', cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}person`, @@ -1090,9 +1062,6 @@ module('Integration | realm', function (hooks) { cardThumbnailURL: null, cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}booking`, @@ -1131,9 +1100,6 @@ module('Integration | realm', function (hooks) { posts: [], cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}booking`, @@ -1269,7 +1235,6 @@ module('Integration | realm', function (hooks) { type: 'card', }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -1300,9 +1265,6 @@ module('Integration | realm', function (hooks) { cardTitle: 'Hassan Abdel-Rahman', cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}person`, @@ -1329,7 +1291,6 @@ module('Integration | realm', function (hooks) { }, relationships: { owner: { links: { self: null } }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -1359,7 +1320,6 @@ module('Integration | realm', function (hooks) { relationships: { 'pets.0': { links: { self: `./dir/van-gogh` } }, friend: { links: { self: `./dir/friend` } }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -1501,7 +1461,6 @@ module('Integration | realm', function (hooks) { type: 'card', }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -1550,9 +1509,6 @@ module('Integration | realm', function (hooks) { data: { id: `${testRealmURL}jackie`, attributes: { firstName: 'Jackie' }, - relationships: { - pets: { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}pet-person`, @@ -1616,8 +1572,6 @@ module('Integration | realm', function (hooks) { links: { self: `./dir/van-gogh` }, data: { id: `${testRealmURL}dir/van-gogh`, type: 'card' }, }, - friend: { links: { self: null } }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -1727,17 +1681,11 @@ module('Integration | realm', function (hooks) { links: { self: `./2` }, data: { id: `${testRealmURL}2`, type: 'card' }, }, - 'cardInfo.theme': { links: { self: null } }, }); assert.deepEqual( JSON.parse((adapter.files.contents['1.json'] as any).content).data .relationships, { - 'cardInfo.theme': { - links: { - self: null, - }, - }, 'inners.0.other': { links: { self: './2', @@ -1837,11 +1785,6 @@ module('Integration | realm', function (hooks) { cardThumbnailURL: null, cardInfo, }, - relationships: { - pets: { links: { self: null } }, - friend: { links: { self: null } }, - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}pet-person`, @@ -1972,7 +1915,6 @@ module('Integration | realm', function (hooks) { type: 'card', }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -2073,9 +2015,6 @@ module('Integration | realm', function (hooks) { data: { type: 'card', relationships: { - pets: { - links: { self: null }, - }, friend: { links: { self: `${testRealmURL}dir/different-friend` }, }, @@ -2108,9 +2047,6 @@ module('Integration | realm', function (hooks) { cardInfo, }, relationships: { - pets: { - links: { self: null }, - }, friend: { links: { self: `./dir/different-friend` }, data: { @@ -2118,7 +2054,13 @@ module('Integration | realm', function (hooks) { type: 'card', }, }, - 'cardInfo.theme': { links: { self: null } }, + 'pets.0': { + links: { self: `./dir/van-gogh` }, + data: { + id: `${testRealmURL}dir/van-gogh`, + type: 'card', + }, + }, }, meta: { adoptsFrom: { @@ -2249,7 +2191,6 @@ module('Integration | realm', function (hooks) { id: `${testRealmURL}dir/mariko`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -2282,9 +2223,6 @@ module('Integration | realm', function (hooks) { cardThumbnailURL: null, cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `${testModuleRealm}person`, @@ -2322,7 +2260,6 @@ module('Integration | realm', function (hooks) { self: `./mariko`, }, }, - 'cardInfo.theme': { links: { self: null } }, }, meta: { adoptsFrom: { @@ -2439,9 +2376,6 @@ module('Integration | realm', function (hooks) { cardTitle: 'Untitled Card', cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `../driver`, @@ -2484,9 +2418,6 @@ module('Integration | realm', function (hooks) { }, cardInfo, }, - relationships: { - 'cardInfo.theme': { links: { self: null } }, - }, meta: { adoptsFrom: { module: `../driver`, @@ -3156,11 +3087,6 @@ module('Integration | realm', function (hooks) { `${testRealmURL}dir/mariko`, 'mango owner id', ); - assert.strictEqual( - mango.relationships['cardInfo.theme'].links.self, - null, - 'mango theme is null', - ); assert.strictEqual( mango.meta.resourceCreatedAt, mangoCreatedAt, @@ -3174,11 +3100,6 @@ module('Integration | realm', function (hooks) { 'Mariko Abdel-Rahman', 'mariko fullName', ); - assert.strictEqual( - mariko.relationships['cardInfo.theme'].links.self, - null, - 'mariko theme is null', - ); assert.strictEqual( mariko.meta.resourceCreatedAt, marikoCreatedAt, @@ -3197,11 +3118,6 @@ module('Integration | realm', function (hooks) { `${testModuleRealm}hassan`, 'vanGogh owner id', ); - assert.strictEqual( - vanGogh.relationships['cardInfo.theme'].links.self, - null, - 'vanGogh theme is null', - ); assert.strictEqual( vanGogh.meta.resourceCreatedAt, vanGoghCreatedAt, @@ -3210,11 +3126,6 @@ module('Integration | realm', function (hooks) { let hassan = included.find((r) => r.id === `${testModuleRealm}hassan`); assert.ok(hassan, 'hassan (cross-realm linksTo target) is included'); - assert.strictEqual( - hassan.relationships['cardInfo.theme'].links.self, - null, - 'included hassan theme is null', - ); }); test('included card uses correct module path when realm is mounted', async function (assert) { diff --git a/packages/host/tests/unit/index-writer-test.ts b/packages/host/tests/unit/index-writer-test.ts index c2c64ac684..1b3675b835 100644 --- a/packages/host/tests/unit/index-writer-test.ts +++ b/packages/host/tests/unit/index-writer-test.ts @@ -9,6 +9,8 @@ import { internalKeyFor, baseCardRef, coerceTypes, + mergeErrorDetail, + mergeErrorsByGeneration, ri, rri, type LooseCardResource, @@ -16,6 +18,7 @@ import { type BoxelIndexTable, type CardResource, type RealmResourceIdentifier, + type SerializedError, } from '@cardstack/runtime-common'; import { CachingDefinitionLookup } from '@cardstack/runtime-common/definition-lookup'; import { VirtualNetwork } from '@cardstack/runtime-common/virtual-network'; @@ -2822,3 +2825,103 @@ module('Unit | index-writer', function (hooks) { }); }); }); + +module('Unit | index-writer | error doc merge', function () { + function err( + message: string, + extra: Partial = {}, + ): SerializedError { + return { message, status: 500, additionalErrors: null, ...extra }; + } + + test('a higher-generation error doc supersedes a lower-generation one', function (assert) { + let recent = err('recent', { + id: 'x', + additionalErrors: [err('recent-dep')], + }); + let stale = err('stale', { id: 'x', additionalErrors: [err('stale-dep')] }); + + assert.strictEqual( + mergeErrorsByGeneration(recent, 5, stale, 4).message, + 'recent', + 'the newer generation wins when it is the first argument', + ); + assert.strictEqual( + mergeErrorsByGeneration(stale, 4, recent, 5).message, + 'recent', + 'the newer generation wins when it is the second argument', + ); + assert.deepEqual( + mergeErrorsByGeneration(recent, 5, stale, 4).additionalErrors, + [err('recent-dep')], + 'the stale document contributes nothing — it is not merged in', + ); + }); + + test('equal-generation error docs merge their detail, keeping the primary message', function (assert) { + let primary = err('primary message', { + id: 'card', + deps: ['dep-a'], + additionalErrors: [err('existing-dep', { id: 'existing' })], + }); + let secondary = err('secondary message', { + id: 'module', + deps: ['dep-b'], + additionalErrors: [err('missing middle-field', { id: 'middle' })], + }); + + let merged = mergeErrorsByGeneration(primary, 3, secondary, 3); + + assert.strictEqual( + merged.message, + 'primary message', + 'the primary envelope message is preserved', + ); + let messages = (merged.additionalErrors ?? []).map((e) => e.message); + assert.ok( + messages.includes('existing-dep'), + "the primary's own additionalErrors are kept", + ); + assert.ok( + messages.includes('missing middle-field'), + "the secondary's nested dependency detail is folded in flat", + ); + assert.ok( + messages.includes('secondary message'), + "the secondary's own top-level error is folded in as a flat entry", + ); + assert.deepEqual(merged.deps, ['dep-a', 'dep-b'], 'deps are unioned'); + }); + + test('merging an error doc identical to the primary adds nothing', function (assert) { + let primary = err('same failure', { id: 'x', status: 500 }); + let secondary = err('same failure', { id: 'x', status: 500 }); + assert.strictEqual( + mergeErrorDetail(primary, secondary).additionalErrors, + null, + 'a secondary that matches the primary (id, message, status) is deduped away', + ); + }); + + test("the secondary's nested detail is folded flat, not nested", function (assert) { + let primary = err('primary', { id: 'p' }); + let secondary = err('secondary', { + id: 's', + additionalErrors: [err('leaf detail', { id: 'leaf' })], + }); + let merged = mergeErrorDetail(primary, secondary); + assert.deepEqual( + (merged.additionalErrors ?? []).map((e) => e.message), + ['secondary', 'leaf detail'], + 'both the secondary envelope and its nested detail sit at the top level', + ); + let secondaryEntry = (merged.additionalErrors ?? []).find( + (e) => e.id === 's', + ); + assert.strictEqual( + secondaryEntry.additionalErrors, + null, + 'the folded secondary entry does not carry its own nested list', + ); + }); +}); diff --git a/packages/realm-server/prerender/prerender-app.ts b/packages/realm-server/prerender/prerender-app.ts index ec85d5492a..2f79a87fe0 100644 --- a/packages/realm-server/prerender/prerender-app.ts +++ b/packages/realm-server/prerender/prerender-app.ts @@ -7,6 +7,7 @@ import { Deferred, type AffinityType, logger, + type PrerenderVisitType, type RenderRouteOptions, type ModuleRenderResponse, type RunCommandResponse, @@ -686,6 +687,16 @@ export function buildPrerenderApp(options: { : {}; let fileData = attrs.fileData; let types = attrs.types; + let rawVisitType = attrs.visitType; + let visitType: PrerenderVisitType | undefined = + rawVisitType === 'index' || rawVisitType === 'prerender-html' + ? rawVisitType + : undefined; + let cardTypes = Array.isArray(attrs.cardTypes) + ? (attrs.cardTypes as unknown[]).filter( + (t): t is string => typeof t === 'string', + ) + : undefined; let isNonEmptyString = (value: unknown): value is string => typeof value === 'string' && value.trim().length > 0; @@ -703,6 +714,12 @@ export function buildPrerenderApp(options: { .filter(({ value }) => !isNonEmptyString(value)) .map(({ name }) => name); + // A visitType value the server doesn't recognize would silently run + // the fused visit; reject instead so a caller skew is loud. + if (rawVisitType != null && visitType === undefined) { + missing.push(`visitType ('index' or 'prerender-html')`); + } + // At least one pass must be requested if ( !renderOptions.fileExtract && @@ -716,9 +733,17 @@ export function buildPrerenderApp(options: { // the composite can chain the extract's resource into render. When // fileExtract isn't requested AND fileData isn't supplied, reject — // the host route model hook requires fileData to populate its model. - if (renderOptions.fileRender && !fileData && !renderOptions.fileExtract) { + // A prerender-html visit never runs the extract, so for it fileData + // is required outright. + if ( + renderOptions.fileRender && + !fileData && + (!renderOptions.fileExtract || visitType === 'prerender-html') + ) { missing.push( - 'fileData (required when fileRender pass is requested without fileExtract)', + visitType === 'prerender-html' + ? 'fileData (required when a prerender-html visit requests fileRender)' + : 'fileData (required when fileRender pass is requested without fileExtract)', ); } // Chaining fileExtract → fileRender also needs fileDefCodeRef so the @@ -737,7 +762,7 @@ export function buildPrerenderApp(options: { let priority = parsePriority(attrs); log.debug( - `received visit prerender request ${rawUrl}: affinityType=${rawAffinityType} affinityValue=${rawAffinityValue} realm=${rawRealm} priority=${priority ?? 0} options=${JSON.stringify(renderOptions)}`, + `received visit prerender request ${rawUrl}: affinityType=${rawAffinityType} affinityValue=${rawAffinityValue} realm=${rawRealm} visitType=${visitType ?? 'fused'} priority=${priority ?? 0} options=${JSON.stringify(renderOptions)}`, ); if (missing.length > 0) { ctxt.status = 400; @@ -782,9 +807,11 @@ export function buildPrerenderApp(options: { realm, url, auth, + ...(visitType ? { visitType } : {}), renderOptions, ...(fileData ? { fileData } : {}), ...(Array.isArray(types) ? { types } : {}), + ...(cardTypes?.length ? { cardTypes } : {}), ...(batchId ? { batchId } : {}), ...(priority !== undefined ? { priority } : {}), ...(jobId ? { jobId } : {}), diff --git a/packages/realm-server/prerender/prerenderer.ts b/packages/realm-server/prerender/prerenderer.ts index 2481b6974a..078238f61c 100644 --- a/packages/realm-server/prerender/prerenderer.ts +++ b/packages/realm-server/prerender/prerenderer.ts @@ -654,9 +654,11 @@ export class Prerenderer { realm, url, auth, + visitType, renderOptions, fileData, types, + cardTypes, opts, priority, jobId, @@ -712,10 +714,12 @@ export class Prerenderer { realm, url, auth, + visitType, opts, renderOptions: attemptOptions, fileData, types, + cardTypes, priority, jobId, signal, @@ -745,10 +749,12 @@ export class Prerenderer { realm, url, auth, + visitType, opts, renderOptions: attemptOptions, fileData, types, + cardTypes, priority, jobId, signal, diff --git a/packages/realm-server/prerender/remote-prerenderer.ts b/packages/realm-server/prerender/remote-prerenderer.ts index 2c8f7d1699..16adf6c8f4 100644 --- a/packages/realm-server/prerender/remote-prerenderer.ts +++ b/packages/realm-server/prerender/remote-prerenderer.ts @@ -231,9 +231,11 @@ export function createRemotePrerenderer( realm, url, auth, + visitType, renderOptions, fileData, types, + cardTypes, batchId, priority, jobId, @@ -247,9 +249,11 @@ export function createRemotePrerenderer( realm, url, auth, + ...(visitType ? { visitType } : {}), renderOptions: renderOptions ?? {}, ...(fileData ? { fileData } : {}), ...(types ? { types } : {}), + ...(cardTypes ? { cardTypes } : {}), ...(batchId ? { batchId } : {}), ...(priority !== undefined ? { priority } : {}), ...(jobId ? { jobId } : {}), diff --git a/packages/realm-server/prerender/render-runner.ts b/packages/realm-server/prerender/render-runner.ts index 32ff21419a..abf1fe1926 100644 --- a/packages/realm-server/prerender/render-runner.ts +++ b/packages/realm-server/prerender/render-runner.ts @@ -46,6 +46,7 @@ import { type RenderProfileContext, } from './utils.ts'; import { randomUUID } from 'crypto'; +import type { Page } from 'puppeteer'; const log = logger('prerenderer'); const reproduceLog = logger('prerenderer-reproduce'); @@ -727,16 +728,24 @@ export class RenderRunner { // fileExtract/cardRender/fileRender passes the caller requests, returning a // union response. Passes execute in VISIT_PASS_ORDER and short-circuit when // the page becomes unusable (eviction or auth failure). + // + // `visitType` bifurcates the pass internals along the search-doc/HTML seam + // (see PrerenderVisitType): an 'index' visit runs the extract, the card's + // icon + meta and the file's icon — never the `html` route; a + // 'prerender-html' visit runs only the `html` route formats + markdown for + // the card and file renderings. No `visitType` runs the fused union. async prerenderVisitAttempt({ affinityType, affinityValue, realm, url, auth, + visitType, opts, renderOptions, fileData, types, + cardTypes, priority, jobId, signal, @@ -752,15 +761,21 @@ export class RenderRunner { pool: PoolInfo; }> { let affinityKey = toAffinityKey({ affinityType, affinityValue }); + // Which halves of the bifurcated visit run. The fused visit (no + // visitType) runs both. + let runIndexSteps = visitType !== 'prerender-html'; + let runHtmlSteps = visitType !== 'index'; let requested = { - fileExtract: Boolean(renderOptions?.fileExtract), + // The extract belongs to the index half; a prerender-html visit never + // runs it even when the flag rides along on shared render options. + fileExtract: Boolean(renderOptions?.fileExtract) && runIndexSteps, cardRender: Boolean(renderOptions?.cardRender), fileRender: Boolean(renderOptions?.fileRender), }; log.info( - `visit prerender url=${url} affinity=${affinityKey} realm=${realm} passes=${VISIT_PASS_ORDER.filter( - (p) => requested[p], - ).join(',')}`, + `visit prerender url=${url} affinity=${affinityKey} realm=${realm} visitType=${ + visitType ?? 'fused' + } passes=${VISIT_PASS_ORDER.filter((p) => requested[p]).join(',')}`, ); const { page, reused, launchMs, waits, pageId, release } = @@ -1034,13 +1049,17 @@ export class RenderRunner { simulateTimeoutMs: opts?.simulateTimeoutMs, timeoutMs: opts?.timeoutMs, }; - reproduceLog.debug( - `manually visit prerendered url ${url} at: ${this.#boxelHostURL}/render/${encodeURIComponent(url)}/${nonce}/${optionsSegment}/html/isolated/0 with boxel-session = ${auth}`, - ); + if (runHtmlSteps) { + reproduceLog.debug( + `manually visit prerendered url ${url} at: ${this.#boxelHostURL}/render/${encodeURIComponent(url)}/${nonce}/${optionsSegment}/html/isolated/0 with boxel-session = ${auth}`, + ); + } let cardError: RenderError | undefined; let cardShortCircuit = false; let isolatedHTML: string | null = null; + let iconHTML: string | null = null; + let capturedDeps: string[] | null = null; let applyStepError = (stepError: RenderError, evicted: boolean) => { cardError = cardError ?? stepError; markTimeout(stepError); @@ -1074,34 +1093,76 @@ export class RenderRunner { return; }; - let isolatedResult = await withTimeout( - page, - async () => { - await transitionTo( - page, - 'render.html', - url, - nonce, - serializedOptions, - 'isolated', - '0', - ); - return await renderHTML(page, 'isolated', 0, captureOptions); - }, - opts?.timeoutMs, - this.#profileContext(affinityKey, url, 'card isolated/0', jobId), - ); - if (isRenderError(isolatedResult)) { - cardShortCircuit = true; - let renderError = isolatedResult as RenderError; - let evicted = await this.#maybeEvict( - affinityKey, - 'visit card isolated render', - renderError, + if (runHtmlSteps) { + let isolatedResult = await withTimeout( + page, + async () => { + await transitionTo( + page, + 'render.html', + url, + nonce, + serializedOptions, + 'isolated', + '0', + ); + return await renderHTML(page, 'isolated', 0, captureOptions); + }, + opts?.timeoutMs, + this.#profileContext(affinityKey, url, 'card isolated/0', jobId), ); - applyStepError(renderError, evicted); + if (isRenderError(isolatedResult)) { + cardShortCircuit = true; + let renderError = isolatedResult as RenderError; + let evicted = await this.#maybeEvict( + affinityKey, + 'visit card isolated render', + renderError, + ); + applyStepError(renderError, evicted); + } else { + isolatedHTML = isolatedResult as string; + } + if (visitType === 'prerender-html' && !cardShortCircuit) { + // The card's runtime deps normally ride on render.meta, which a + // prerender-html visit never runs. The render route publishes the + // settle-time dependency snapshot (unresolved form) on a global; + // read it here so the HTML rendering still reports what it + // pulled in — the indexing job unions this with the index + // visit's meta deps. + capturedDeps = await this.#readCapturedDeps(page); + } } else { - isolatedHTML = isolatedResult as string; + // An index visit never touches the html route, so the icon render + // is its entry into the render app; subsequent steps are in-page + // child transitions. + let iconResult = await withTimeout( + page, + async () => { + await transitionTo( + page, + 'render.icon', + url, + nonce, + serializedOptions, + ); + return await renderIcon(page, captureOptions); + }, + opts?.timeoutMs, + this.#profileContext(affinityKey, url, 'card icon', jobId), + ); + if (isRenderError(iconResult)) { + cardShortCircuit = true; + let renderError = iconResult as RenderError; + let evicted = await this.#maybeEvict( + affinityKey, + 'visit card icon render', + renderError, + ); + applyStepError(renderError, evicted); + } else { + iconHTML = iconResult as string; + } } let emptyMeta: PrerenderMeta = { @@ -1115,12 +1176,11 @@ export class RenderRunner { let typesForAncestors: PrerenderTypes = { types: null }; let headHTML: string | null = null; let atomHTML: string | null = null; - let iconHTML: string | null = null; let embeddedHTML: Record | null = null; let fittedHTML: Record | null = null; let markdown: string | null = null; - if (!cardShortCircuit) { + if (!cardShortCircuit && runHtmlSteps) { const formatSteps = [ { name: 'visit card head render', @@ -1136,13 +1196,19 @@ export class RenderRunner { atomHTML = v; }, }, - { - name: 'visit card icon render', - cb: () => renderIcon(page, captureOptions), - assign: (v: string) => { - iconHTML = v; - }, - }, + // The icon belongs to the index half; the fused visit renders it + // between atom and markdown. + ...(runIndexSteps + ? [ + { + name: 'visit card icon render', + cb: () => renderIcon(page, captureOptions), + assign: (v: string) => { + iconHTML = v; + }, + }, + ] + : []), { name: 'visit card markdown render', cb: () => renderHTML(page, 'markdown', 0, captureOptions), @@ -1158,21 +1224,27 @@ export class RenderRunner { } } - // First pass is the lightweight /types route — just the type - // chain the ancestor renders below need. The full render.meta - // (serialized + searchDoc + deps + displayNames) runs once - // afterwards. - if (!cardShortCircuit) { - let typesResult = await runTimedStep( - 'visit card render.types', - () => renderTypes(page, captureOptions), - ); - if (typesResult !== undefined) { - typesForAncestors = typesResult; + // The ancestor type chain drives the fitted/embedded format renders. + // A caller that already holds the chain passes it in (the indexing + // job forwards the index visit's types); otherwise the lightweight + // /types route resolves it — just the chain, not the full + // render.meta (serialized + searchDoc + deps + displayNames), which + // belongs to the index half. + if (!cardShortCircuit && runHtmlSteps) { + if (cardTypes?.length) { + typesForAncestors = { types: cardTypes }; + } else { + let typesResult = await runTimedStep( + 'visit card render.types', + () => renderTypes(page, captureOptions), + ); + if (typesResult !== undefined) { + typesForAncestors = typesResult; + } } } - if (!cardShortCircuit && typesForAncestors.types) { + if (!cardShortCircuit && runHtmlSteps && typesForAncestors.types) { const ancestorSteps = [ { name: 'visit card fitted render', @@ -1211,7 +1283,7 @@ export class RenderRunner { } } - if (!cardShortCircuit) { + if (!cardShortCircuit && runIndexSteps) { let finalMetaResult = await runTimedStep( 'visit card render.meta', () => renderMeta(page, captureOptions), @@ -1223,6 +1295,7 @@ export class RenderRunner { let cardResponse: RenderResponse = { ...(meta as PrerenderMeta), + ...(capturedDeps ? { deps: capturedDeps } : {}), ...(cardError ? { error: cardError } : {}), iconHTML, isolatedHTML, @@ -1330,49 +1403,81 @@ export class RenderRunner { } }; - let isolatedResult = await withTimeout( - page, - async () => { - await transitionTo( - page, - 'render.html', - url, - nonce, - serializedOptions, - 'isolated', - '0', - ); - return await captureResult(page, 'innerHTML', captureOptions); - }, - opts?.timeoutMs, - this.#profileContext(affinityKey, url, 'file isolated/0', jobId), - ); - if (isRenderError(isolatedResult)) { - let renderError = isolatedResult as RenderError; - let evicted = await this.#maybeEvict( - affinityKey, - 'visit file isolated render', - renderError, + if (runHtmlSteps) { + let isolatedResult = await withTimeout( + page, + async () => { + await transitionTo( + page, + 'render.html', + url, + nonce, + serializedOptions, + 'isolated', + '0', + ); + return await captureResult(page, 'innerHTML', captureOptions); + }, + opts?.timeoutMs, + this.#profileContext(affinityKey, url, 'file isolated/0', jobId), ); - applyStepError(renderError, evicted); - } else { - let capture = isolatedResult as RenderCapture; - if (capture.status === 'ready') { - isolatedHTML = capture.value; - } else { - let capErr = this.#captureToError(capture); + if (isRenderError(isolatedResult)) { + let renderError = isolatedResult as RenderError; let evicted = await this.#maybeEvict( affinityKey, 'visit file isolated render', - capErr, + renderError, ); - if (capErr) { - applyStepError(capErr, evicted); + applyStepError(renderError, evicted); + } else { + let capture = isolatedResult as RenderCapture; + if (capture.status === 'ready') { + isolatedHTML = capture.value; + } else { + let capErr = this.#captureToError(capture); + let evicted = await this.#maybeEvict( + affinityKey, + 'visit file isolated render', + capErr, + ); + if (capErr) { + applyStepError(capErr, evicted); + } } } + } else { + // The file's icon belongs to the index half, and an index visit + // never touches the html route — so the icon render is its entry + // into the render app for this file. + let iconResult = await withTimeout( + page, + async () => { + await transitionTo( + page, + 'render.icon', + url, + nonce, + serializedOptions, + ); + return await renderIcon(page, captureOptions); + }, + opts?.timeoutMs, + this.#profileContext(affinityKey, url, 'file icon', jobId), + ); + if (isRenderError(iconResult)) { + let renderError = iconResult as RenderError; + let evicted = await this.#maybeEvict( + affinityKey, + 'visit file icon render', + renderError, + ); + applyStepError(renderError, evicted); + } else { + iconHTML = iconResult as string; + } } - if (!fileShortCircuit) { + if (!fileShortCircuit && runHtmlSteps) { let headHTMLResult = await this.#step( affinityKey, 'visit file head render', @@ -1391,7 +1496,7 @@ export class RenderRunner { } } - if (!fileShortCircuit) { + if (!fileShortCircuit && runHtmlSteps) { let steps: Array<{ name: string; cb: () => Promise | RenderError>; @@ -1429,29 +1534,31 @@ export class RenderRunner { ); } - steps.push( - { - name: 'visit file atom render', - cb: () => renderHTML(page, 'atom', 0, captureOptions), - assign: (v) => { - atomHTML = v as string; - }, + steps.push({ + name: 'visit file atom render', + cb: () => renderHTML(page, 'atom', 0, captureOptions), + assign: (v) => { + atomHTML = v as string; }, - { + }); + if (runIndexSteps) { + // The icon belongs to the index half; the fused visit renders + // it between atom and markdown. + steps.push({ name: 'visit file icon render', cb: () => renderIcon(page, captureOptions), assign: (v) => { iconHTML = v as string; }, + }); + } + steps.push({ + name: 'visit file markdown render', + cb: () => renderHTML(page, 'markdown', 0, captureOptions), + assign: (v) => { + markdown = v as string; }, - { - name: 'visit file markdown render', - cb: () => renderHTML(page, 'markdown', 0, captureOptions), - assign: (v) => { - markdown = v as string; - }, - }, - ); + }); for (let step of steps) { if (fileShortCircuit) break; @@ -1572,6 +1679,26 @@ export class RenderRunner { return status === 401 || status === 403; } + // Settle-time runtime-dependency snapshot the render route publishes on + // `globalThis.__boxelRenderCapturedDeps` (already in unresolved/prefix + // form). Best-effort: a page that died mid-read reports null rather than + // failing the visit. + async #readCapturedDeps(page: Page): Promise { + try { + let deps = await page.evaluate( + () => + (globalThis as { __boxelRenderCapturedDeps?: unknown }) + .__boxelRenderCapturedDeps ?? null, + ); + if (!Array.isArray(deps)) { + return null; + } + return deps.filter((dep): dep is string => typeof dep === 'string'); + } catch (_e) { + return null; + } + } + async #step( affinityKey: string, step: string, diff --git a/packages/realm-server/tests/card-endpoints-test.ts b/packages/realm-server/tests/card-endpoints-test.ts index ba62693bf2..0ddf4bcabb 100644 --- a/packages/realm-server/tests/card-endpoints-test.ts +++ b/packages/realm-server/tests/card-endpoints-test.ts @@ -185,18 +185,6 @@ module(basename(import.meta.filename), function () { cardDescription: null, cardThumbnailURL: null, }, - relationships: { - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri(`./person`), @@ -988,18 +976,6 @@ module(basename(import.meta.filename), function () { realmInfo: testRealmInfo, realmURL: testRealmHref, }, - relationships: { - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, links: { self: `${testRealmHref}person-1`, }, @@ -1358,6 +1334,105 @@ module(basename(import.meta.filename), function () { ); }); + test('an explicitly-null relationship is preserved and an absent one omitted, in both the source and the served card+json', async function (assert) { + let realmEventTimestampStart = Date.now(); + + // `Friend.friend` (linksTo) and `Friend.friends` (linksToMany) are + // both non-searchable. Author `friend` explicitly null and leave + // `friends` absent. Serialization keys on whether the card actually + // has the relationship — an authored empty (`{ self: null }`) vs a + // never-set link — independent of searchability, so both the written + // source and the served card+json keep `friend` as `{ self: null }` + // and omit `friends`. (`friend` is even rendered in the isolated + // template, yet a render doesn't author a link — reading it doesn't + // mark it "used" — so it stays omitted unless actually set.) + let response = await request + .post('/') + .send({ + data: { + type: 'card', + attributes: { + firstName: 'Hassan', + }, + relationships: { + friend: { + links: { + self: null, + }, + }, + }, + meta: { + adoptsFrom: { + module: rri('https://localhost:4202/node-test/friend'), + name: 'Friend', + }, + }, + }, + } as LooseSingleCardDocument) + .set('Accept', 'application/vnd.card+json'); + + let incrementalEventContent = await expectIncrementalIndexEvent( + testRealmHref, + realmEventTimestampStart, + { + assert, + getMessagesSince, + realm: testRealmHref, + type: 'Friend', + timeout: 5000, + }, + ); + let id = incrementalEventContent.invalidations[0].split('/').pop()!; + + assert.strictEqual( + response.status, + 201, + `HTTP 201 status: ${response.text}`, + ); + + // The written source persists the card's relationships as authored: + // the explicitly-null `friend` survives as `{ self: null }` while the + // never-authored `friends` is absent. + let cardFile = join( + dir.name, + 'realm_server_1', + 'test', + 'Friend', + `${id}.json`, + ); + assert.ok(existsSync(cardFile), `card json ${cardFile} exists`); + let source = readJSONSync(cardFile) as LooseSingleCardDocument; + assert.deepEqual( + source.data.relationships, + { friend: { links: { self: null } } }, + 'source keeps the explicitly-null friend link and omits the absent friends link', + ); + + // The served card+json (the indexed pristine doc) keeps the same + // distinction: `friend` was authored empty, so it round-trips as + // `{ self: null }`; `friends` was never set, so it is omitted. This is + // data fidelity, independent of searchability. + let getResponse = await request + .get(`/Friend/${id}`) + .set('Accept', 'application/vnd.card+json'); + assert.strictEqual( + getResponse.status, + 200, + `HTTP 200 status: ${getResponse.text}`, + ); + let served = getResponse.body as SingleCardDocument; + assert.deepEqual( + served.data.relationships?.friend, + { links: { self: null } }, + 'served card+json preserves the explicitly-null friend link', + ); + assert.strictEqual( + served.data.relationships?.friends, + undefined, + 'served card+json omits the absent friends link', + ); + }); + test('creates card instances when it encounters "lid" in the request', async function (assert) { let response = await request .post('/') @@ -1623,16 +1698,6 @@ module(basename(import.meta.filename), function () { id: `${testRealmHref}Friend/local-id-1`, }, }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -1686,21 +1751,6 @@ module(basename(import.meta.filename), function () { type: 'card', }, }, - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -1719,23 +1769,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('https://localhost:4202/node-test/friend'), @@ -1753,23 +1786,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('https://localhost:4202/node-test/friend'), @@ -1825,21 +1841,6 @@ module(basename(import.meta.filename), function () { type: 'card', }, }, - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -1874,23 +1875,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('https://localhost:4202/node-test/friend'), @@ -1908,23 +1892,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('https://localhost:4202/node-test/friend'), @@ -1962,23 +1929,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { name: 'Friend', @@ -2019,23 +1969,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { name: 'Friend', @@ -2127,8 +2060,15 @@ module(basename(import.meta.filename), function () { firstName: 'Hassan', }, relationships: { + // The written source records the explicitly-nulled cross- + // realm `friend` link — the write path persists the card's + // own relationships as authored, and an authored + // `{ self: null }` is preserved (the served card+json keeps + // it too; only never-authored links are omitted). friend: { - links: { self: null }, + links: { + self: null, + }, }, }, meta: { @@ -2238,23 +2178,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { name: 'Friend', @@ -2429,18 +2352,6 @@ module(basename(import.meta.filename), function () { firstName: 'Van Gogh', cardInfo, }, - relationships: { - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri(`./person`), @@ -2753,10 +2664,10 @@ module(basename(import.meta.filename), function () { 'Recovered', 'card file updated from error state', ); - assert.deepEqual( + assert.strictEqual( card.data.relationships?.['cardInfo.theme'], - { links: { self: null } }, - 'relationships from pristine doc are preserved', + undefined, + 'the unset, non-searchable base-card link is not persisted (not in a searchable path and never set)', ); }); @@ -2957,16 +2868,6 @@ module(basename(import.meta.filename), function () { self: './Friend/local-id-1', }, }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -3113,16 +3014,6 @@ module(basename(import.meta.filename), function () { id: `${testRealmHref}Friend/local-id-1`, }, }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -3176,21 +3067,6 @@ module(basename(import.meta.filename), function () { type: 'card', }, }, - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -3209,23 +3085,6 @@ module(basename(import.meta.filename), function () { cardDescription: null, cardThumbnailURL: null, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('./friend'), @@ -3243,23 +3102,6 @@ module(basename(import.meta.filename), function () { cardDescription: null, cardThumbnailURL: null, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('./friend'), @@ -3315,21 +3157,6 @@ module(basename(import.meta.filename), function () { type: 'card', }, }, - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -3364,23 +3191,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('../friend'), @@ -3398,23 +3208,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('../friend'), @@ -3452,23 +3245,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { name: 'Friend', @@ -3509,23 +3285,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { name: 'Friend', @@ -3640,16 +3399,6 @@ module(basename(import.meta.filename), function () { self: './FriendWithUsedLink/local-id-1', }, }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -3730,16 +3479,6 @@ module(basename(import.meta.filename), function () { id: `${testRealmHref}FriendWithUsedLink/local-id-1`, }, }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { @@ -3776,23 +3515,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri( @@ -3831,23 +3553,6 @@ module(basename(import.meta.filename), function () { cardThumbnailURL: null, cardInfo, }, - relationships: { - friend: { - links: { - self: null, - }, - }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { name: 'FriendWithUsedLink', @@ -3941,16 +3646,6 @@ module(basename(import.meta.filename), function () { friend: { links: { self: './jade' }, }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, meta: { adoptsFrom: { diff --git a/packages/realm-server/tests/card-source-endpoints-test.ts b/packages/realm-server/tests/card-source-endpoints-test.ts index f2e1ea5ea2..be9c720ef5 100644 --- a/packages/realm-server/tests/card-source-endpoints-test.ts +++ b/packages/realm-server/tests/card-source-endpoints-test.ts @@ -943,18 +943,6 @@ module(basename(import.meta.filename), function () { field2a: 'c', cardInfo, }, - relationships: { - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, meta: { adoptsFrom: { module: rri('../test-card'), diff --git a/packages/realm-server/tests/indexing-test.ts b/packages/realm-server/tests/indexing-test.ts index 4c81e50f63..e9dcbe24be 100644 --- a/packages/realm-server/tests/indexing-test.ts +++ b/packages/realm-server/tests/indexing-test.ts @@ -775,21 +775,17 @@ module(basename(import.meta.filename), function () { assert.deepEqual( hassan.doc.data.relationships, { + // Only `pet` appears: it is the relationship this card actually + // has (a set target). The base-card `cardInfo.theme` / + // `cardInfo.cardThumbnail` links are never authored here, so they + // drop from the pristine doc — this filtering is data-driven, not + // searchable-driven (they still appear in the search doc, which + // enumerates every declared field). pet: { links: { self: './ringo', }, }, - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, }, 'doc relationships are correct', ); diff --git a/packages/realm-server/tests/prerendering-test.ts b/packages/realm-server/tests/prerendering-test.ts index a8a2df3aca..a8c874b9b3 100644 --- a/packages/realm-server/tests/prerendering-test.ts +++ b/packages/realm-server/tests/prerendering-test.ts @@ -7422,6 +7422,216 @@ module(basename(import.meta.filename), function () { ); }); + // The bifurcated visits split the fused visit's output along the + // search-doc/HTML seam. The indexing job runs the index visit then the + // prerender-html visit and merges the halves into the same writes a + // fused visit produces — so the two halves must partition the fused + // output exactly: no field produced by both, none produced by neither, + // and every field byte-identical to its fused counterpart. + test('index and prerender-html visits partition the fused output losslessly', async function (assert) { + const cardFileURL = `${realmURL}maple.json`; + const fileDefCodeRef = { + module: baseRRI('json-file-def'), + name: 'JsonFileDef', + }; + let visitArgs = { + affinityType: 'realm' as const, + affinityValue: realmURL, + realm: realmURL, + url: cardFileURL, + auth: auth(), + }; + const htmlFields = [ + 'isolatedHTML', + 'headHTML', + 'atomHTML', + 'embeddedHTML', + 'fittedHTML', + 'markdown', + ] as const; + + let fused = ( + await prerenderer.prerenderVisit({ + ...visitArgs, + renderOptions: { + cardRender: true, + fileExtract: true, + fileRender: true, + fileDefCodeRef, + }, + }) + ).response; + let index = ( + await prerenderer.prerenderVisit({ + ...visitArgs, + visitType: 'index', + renderOptions: { + cardRender: true, + fileExtract: true, + fileRender: true, + fileDefCodeRef, + }, + }) + ).response; + assert.ok( + index.fileExtract?.resource, + 'index visit extract produced a resource to chain into the prerender-html visit', + ); + let html = ( + await prerenderer.prerenderVisit({ + ...visitArgs, + visitType: 'prerender-html', + renderOptions: { + cardRender: true, + fileRender: true, + fileDefCodeRef, + }, + fileData: { + resource: index.fileExtract!.resource!, + fileDefCodeRef, + }, + types: index.fileExtract!.types ?? undefined, + cardTypes: index.card!.types ?? undefined, + }) + ).response; + + // The index visit produces only the search-doc side. + assert.notOk( + index.card?.error, + `index visit card completed without error: ${JSON.stringify(index.card?.error?.error ?? null)}`, + ); + assert.ok(index.card?.serialized, 'index visit serializes the card'); + assert.ok(index.card?.searchDoc, 'index visit builds the search doc'); + assert.ok(index.card?.types?.length, 'index visit reports types'); + assert.ok(index.card?.deps?.length, 'index visit reports deps'); + assert.ok(index.card?.iconHTML, 'index visit renders the card icon'); + assert.ok( + index.fileRender?.iconHTML, + 'index visit renders the file icon', + ); + for (let field of htmlFields) { + assert.strictEqual( + index.card?.[field], + null, + `index visit leaves card ${field} null`, + ); + assert.strictEqual( + index.fileRender?.[field], + null, + `index visit leaves file ${field} null`, + ); + } + + // The prerender-html visit produces only the HTML side. + assert.notOk( + html.card?.error, + `prerender-html visit card completed without error: ${JSON.stringify(html.card?.error?.error ?? null)}`, + ); + assert.notOk(html.fileExtract, 'prerender-html visit skips the extract'); + assert.strictEqual( + html.card?.serialized, + null, + 'prerender-html visit does not serialize the card', + ); + assert.strictEqual( + html.card?.searchDoc, + null, + 'prerender-html visit does not build a search doc', + ); + assert.strictEqual( + html.card?.types, + null, + 'prerender-html visit does not report types', + ); + assert.strictEqual( + html.card?.iconHTML, + null, + 'prerender-html visit does not render the card icon', + ); + assert.strictEqual( + html.fileRender?.iconHTML, + null, + 'prerender-html visit does not render the file icon', + ); + assert.ok( + html.card?.deps?.length, + 'prerender-html visit reports the deps its renders pulled in', + ); + + // Merging the two halves reproduces the fused output. The file's + // isolated HTML is captured as raw innerHTML (card HTML goes through + // the ember-id scrub, file isolated does not), so auto-generated + // `id="emberNN"` values vary per render and are normalized away + // before comparing. + let scrubEmberIds = (value: unknown) => + typeof value === 'string' + ? value.replace(/id="ember\d+"/g, 'id="ember"') + : value; + for (let field of htmlFields) { + assert.deepEqual( + html.card?.[field], + fused.card?.[field], + `card ${field} matches the fused visit`, + ); + assert.deepEqual( + scrubEmberIds(html.fileRender?.[field]), + scrubEmberIds(fused.fileRender?.[field]), + `file ${field} matches the fused visit`, + ); + } + assert.deepEqual( + index.card?.iconHTML, + fused.card?.iconHTML, + 'card icon matches the fused visit', + ); + assert.deepEqual( + index.fileRender?.iconHTML, + fused.fileRender?.iconHTML, + 'file icon matches the fused visit', + ); + assert.deepEqual( + index.card?.serialized, + fused.card?.serialized, + 'serialized doc matches the fused visit', + ); + assert.deepEqual( + index.card?.searchDoc, + fused.card?.searchDoc, + 'search doc matches the fused visit', + ); + assert.deepEqual( + index.card?.displayNames, + fused.card?.displayNames, + 'display names match the fused visit', + ); + assert.deepEqual( + index.card?.types, + fused.card?.types, + 'types match the fused visit', + ); + { + let { deps: fusedDeps, nonce: _n1, ...fusedRest } = fused.fileExtract!; + let { deps: indexDeps, nonce: _n2, ...indexRest } = index.fileExtract!; + assert.deepEqual( + indexRest, + fusedRest, + 'file extract matches the fused visit', + ); + assert.deepEqual( + [...new Set(indexDeps)].sort(), + [...new Set(fusedDeps)].sort(), + 'file extract deps match the fused visit as a set', + ); + } + assert.deepEqual( + [ + ...new Set([...(index.card?.deps ?? []), ...(html.card?.deps ?? [])]), + ].sort(), + [...new Set(fused.card?.deps ?? [])].sort(), + 'union of the two visits card deps matches the fused deps as a set', + ); + }); + test('reuses a single pooled page for all three passes', async function (assert) { const cardFileURL = `${realmURL}maple.json`; // Warm the pool diff --git a/packages/realm-server/tests/realm-endpoints-test.ts b/packages/realm-server/tests/realm-endpoints-test.ts index 7ea851a05a..f5e9e522c3 100644 --- a/packages/realm-server/tests/realm-endpoints-test.ts +++ b/packages/realm-server/tests/realm-endpoints-test.ts @@ -815,18 +815,6 @@ module(basename(import.meta.filename), function () { links: { self: newCardId, }, - relationships: { - 'cardInfo.cardThumbnail': { - links: { - self: null, - }, - }, - 'cardInfo.theme': { - links: { - self: null, - }, - }, - }, }, }); } diff --git a/packages/runtime-common/error.ts b/packages/runtime-common/error.ts index 287297dbc2..de082275e6 100644 --- a/packages/runtime-common/error.ts +++ b/packages/runtime-common/error.ts @@ -235,6 +235,85 @@ function omittedSentinel(count: number): { }; } +// Identity used to dedupe error documents when folding one into another — +// two documents describing the same failure (same target, message, status) +// are the same error even if one rode in via a different path. +function errorDocIdentity(error: { + id?: string | null; + message?: string; + status?: number; +}): string { + return JSON.stringify({ + id: error.id ?? null, + message: error.message ?? null, + status: error.status ?? null, + }); +} + +// Fold `secondary`'s error detail into `primary` without changing which +// document is authoritative: `primary` keeps its message / status / stack +// (the reader-facing text), and `secondary`'s detail is added to +// `primary.additionalErrors` so a dependency error only one document captured +// survives. `secondary`'s own top-level error is added as a flat entry (its +// nested `additionalErrors` are spread in alongside rather than nested), and +// `deps` are unioned. Entries are deduped by (id, message, status). +export function mergeErrorDetail( + primary: SerializedError, + secondary: SerializedError, +): SerializedError { + let merged = Array.isArray(primary.additionalErrors) + ? [...primary.additionalErrors] + : []; + let seen = new Set([ + errorDocIdentity(primary), + ...merged.map(errorDocIdentity), + ]); + let candidates = [ + { ...secondary, additionalErrors: null }, + ...(Array.isArray(secondary.additionalErrors) + ? secondary.additionalErrors + : []), + ]; + for (let candidate of candidates) { + let key = errorDocIdentity(candidate); + if (!seen.has(key)) { + seen.add(key); + merged.push(candidate); + } + } + let deps = + primary.deps || secondary.deps + ? [...new Set([...(primary.deps ?? []), ...(secondary.deps ?? [])])] + : undefined; + return { + ...primary, + additionalErrors: merged.length > 0 ? merged : null, + ...(deps ? { deps } : {}), + }; +} + +// Combine two error documents for the same indexed entry, using their index +// generation as the authority. A higher generation is more recent, so it wins +// outright: the lower-generation document may be stale — the dependency graph +// it recorded may have changed since. At the same generation neither +// supersedes the other (e.g. the two prerender visits of a single indexing +// job): keep `a` as the authoritative envelope and fold `b`'s detail into it +// via `mergeErrorDetail`. +export function mergeErrorsByGeneration( + a: SerializedError, + aGeneration: number, + b: SerializedError, + bGeneration: number, +): SerializedError { + if (aGeneration > bGeneration) { + return a; + } + if (bGeneration > aGeneration) { + return b; + } + return mergeErrorDetail(a, b); +} + export interface CardErrorJSONAPI { id?: string; // 404 errors won't necessarily have an id status: number; diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 334c0239d5..8d87bfb995 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -42,7 +42,7 @@ import { discoverInvalidations, type DiscoverInvalidationsResult, } from './index-runner/discover-invalidations.ts'; -import { visitFileForIndexingFused } from './index-runner/visit-file.ts'; +import { visitFileForIndexing } from './index-runner/visit-file.ts'; import { performCardIndexing } from './index-runner/card-indexer.ts'; import { performFileIndexing } from './index-runner/file-indexer.ts'; @@ -520,7 +520,7 @@ export class IndexRunner { private async tryToVisit(url: URL) { try { - await visitFileForIndexingFused({ + await visitFileForIndexing({ url, realmURL: this.#realmURL, ignoreMap: this.ignoreMap, @@ -966,7 +966,7 @@ export class IndexRunner { lastModified: number; resourceCreatedAt: number; resource: LooseCardResource; - // Render result produced by the fused visit's cardRender pass. + // Merged card result from the file's index + prerender-html visits. renderResult: NonNullable< Parameters[0]['precomputedRenderResult'] >; @@ -1022,9 +1022,9 @@ export class IndexRunner { lastModified: number; resourceCreatedAt: number; hasModulePrerender?: boolean; - // Extract/render results produced by the fused visit's fileExtract / - // fileRender passes. Either may be undefined if the visit chose not to - // run that pass (e.g. fileRender is skipped for module files). + // Extract result from the index visit and merged render result from the + // index + prerender-html visits. Either may be undefined if the visits + // short-circuited before producing it. extractResult?: Parameters< typeof performFileIndexing >[0]['precomputedExtractResult']; diff --git a/packages/runtime-common/index-runner/card-indexer.ts b/packages/runtime-common/index-runner/card-indexer.ts index 4e4ff483d6..dc09f74b88 100644 --- a/packages/runtime-common/index-runner/card-indexer.ts +++ b/packages/runtime-common/index-runner/card-indexer.ts @@ -35,11 +35,11 @@ export interface CardIndexerOptions { realmURL: URL; auth: string; jobInfo: JobInfo; - // Render result from the fused visit's cardRender pass. Always supplied - // by the fused indexer. + // Merged card result from the file's index + prerender-html visits. + // Always supplied by the visit-file indexer. precomputedRenderResult: RenderResponse; - // Timing / diagnostic payload attached to the fused-visit - // response; persisted onto `boxel_index.diagnostics`. + // Timing / diagnostic payload attached to the visit + // responses; persisted onto `boxel_index.diagnostics`. diagnostics?: Diagnostics; dependencyResolver: IndexRunnerDependencyManager; virtualNetwork: VirtualNetwork; diff --git a/packages/runtime-common/index-runner/file-indexer.ts b/packages/runtime-common/index-runner/file-indexer.ts index 2d4a7ee7f5..14faec0405 100644 --- a/packages/runtime-common/index-runner/file-indexer.ts +++ b/packages/runtime-common/index-runner/file-indexer.ts @@ -35,13 +35,13 @@ export interface FileIndexerOptions { realmURL: URL; auth: string; jobInfo: JobInfo; - // Extract / render results from the fused visit. extractResult may be - // undefined if the visit short-circuited before the fileExtract pass ran; - // renderResult may be undefined if the visit skipped fileRender (e.g. for - // module files, which produce HTML via their own module prerender). + // Extract result from the index visit and merged render result from the + // index + prerender-html visits. extractResult may be undefined if the + // visit short-circuited before the fileExtract pass ran; renderResult may + // be undefined if no visit produced a file rendering. precomputedExtractResult: FileExtractResponse | undefined; precomputedRenderResult?: FileRenderResponse; - // Timing / diagnostic payload attached to the fused-visit response; + // Timing / diagnostic payload attached to the visit responses; // persisted onto `boxel_index.diagnostics` for this file's row. diagnostics?: Diagnostics; dependencyResolver: IndexRunnerDependencyManager; @@ -192,17 +192,17 @@ export async function performFileIndexing({ return 'error'; } - // HTML for the file entry comes from the fused visit's fileRender pass - // (when the visit chose to run it — modules skip fileRender since their - // module prerender already produces HTML). + // HTML for the file entry comes from the visits' fileRender passes (icon + // from the index visit, html formats + markdown from the prerender-html + // visit). let renderResult: FileRenderResponse | undefined = precomputedRenderResult; if (renderResult?.error) { logWarn( `${jobIdentity(jobInfo)} file render produced error for ${path}, retaining partial HTML: ${renderResult.error.error?.message}`, ); } - // hasModulePrerender is retained on the options as a hint for callers but - // is no longer acted on here — the fused visit already gates fileRender. + // hasModulePrerender remains on the options as a hint for callers; the visit + // itself gates fileRender, so this path does not act on it. void hasModulePrerender; // A frontmatter parse failure doesn't fail the file (it still indexes diff --git a/packages/runtime-common/index-runner/visit-file.ts b/packages/runtime-common/index-runner/visit-file.ts index 34089f4bba..5d4c746d3f 100644 --- a/packages/runtime-common/index-runner/visit-file.ts +++ b/packages/runtime-common/index-runner/visit-file.ts @@ -8,21 +8,23 @@ import { jobIdentity, unixTime, type Batch, + type Diagnostics, + type FileRenderResponse, type JobInfo, type LocalPath, type LooseCardResource, type Prerenderer, type Reader, type RealmPaths, + type RenderResponse, type RenderRouteOptions, type RenderVisitResponse, - type Diagnostics, } from '../index.ts'; -import { CardError } from '../error.ts'; +import { CardError, mergeErrorsByGeneration } from '../error.ts'; import { resolveFileDefCodeRef } from '../file-def-code-ref.ts'; import type { VirtualNetwork } from '../virtual-network.ts'; -interface VisitFileFusedOptions { +interface VisitFileOptions { url: URL; realmURL: URL; ignoreMap: Map; @@ -52,7 +54,7 @@ interface VisitFileFusedOptions { resourceCreatedAt: number; resource: LooseCardResource; renderResult: NonNullable; - // Timing / diagnostic payload flattened from the fused visit's + // Timing / diagnostic payload flattened from the visits' // `response.meta` (server timings + host-side breadcrumbs + // HTTP requestId). Persisted onto `boxel_index.diagnostics` // so operators can investigate slow renders after the fact. @@ -69,11 +71,22 @@ interface VisitFileFusedOptions { }): Promise; } -// Fused visit: calls prerenderer.prerenderVisit once with whichever of the -// fileExtract/cardRender/fileRender passes are needed for this file, then -// routes the sub-results into the card/file indexers. Replaces up to 3 -// separate prerender HTTP round-trips with a single one. -export async function visitFileForIndexingFused({ +// Visits a file for indexing as two consolidated prerender visits along the +// search-doc/HTML seam, then routes the merged sub-results into the +// card/file indexers: +// +// 1. the index visit — file extract, card meta (search doc / serialized / +// types / display names / deps) and the card + file icons. Never runs +// the html route. +// 2. the prerender-html visit — the html route per format plus markdown +// for the card and the file rendering, chained off the index visit's +// outputs (extract resource → fileData, extract types → FileDef +// ancestor chain, meta types → card ancestor chain). +// +// The merged writes are identical to what a single fused visit produces; +// the seam separates HTML production from search-doc production so each can +// run on its own channel. +export async function visitFileForIndexing({ url, realmURL, ignoreMap, @@ -91,19 +104,19 @@ export async function visitFileForIndexingFused({ logWarn, indexCardWithResult, indexFileWithResults, -}: VisitFileFusedOptions): Promise { +}: VisitFileOptions): Promise { if (isIgnored(realmURL, ignoreMap, url)) { return; } let start = Date.now(); - logDebug(`${jobIdentity(jobInfo)} begin fused visit of file ${url.href}`); + logDebug(`${jobIdentity(jobInfo)} begin visit of file ${url.href}`); let localPath: string; try { localPath = realmPaths.local(url); } catch (_e) { logDebug( - `${jobIdentity(jobInfo)} Fused visit of ${url.href} skipped (different realm than ${realmURL.href})`, + `${jobIdentity(jobInfo)} Visit of ${url.href} skipped (different realm than ${realmURL.href})`, ); return; } @@ -183,7 +196,19 @@ export async function visitFileForIndexingFused({ } } - let renderOptions: RenderRouteOptions = { + let visitArgs = { + affinityType: 'realm' as const, + affinityValue: realmURL.href, + realm: realmURL.href, + url: fileURL, + auth, + batchId, + ...(jobPriority !== undefined ? { priority: jobPriority } : {}), + ...(jobInfo ? { jobId: `${jobInfo.jobId}.${jobInfo.reservationId}` } : {}), + }; + + // The index visit runs first and carries the one-shot clearCache. + let indexRenderOptions: RenderRouteOptions = { fileDefCodeRef, ...(needCardRender ? { cardRender: true } : {}), ...(needFileExtract ? { fileExtract: true } : {}), @@ -193,66 +218,106 @@ export async function visitFileForIndexingFused({ ...(fileContentSize !== undefined ? { fileContentSize } : {}), }; - let visitResponse: RenderVisitResponse; + let indexResponse: RenderVisitResponse; try { - visitResponse = await prerenderer.prerenderVisit({ - affinityType: 'realm', - affinityValue: realmURL.href, - realm: realmURL.href, - url: fileURL, - auth, - renderOptions, - batchId, - ...(jobPriority !== undefined ? { priority: jobPriority } : {}), - ...(jobInfo - ? { jobId: `${jobInfo.jobId}.${jobInfo.reservationId}` } - : {}), + indexResponse = await prerenderer.prerenderVisit({ + ...visitArgs, + visitType: 'index', + renderOptions: indexRenderOptions, }); } catch (err) { logWarn( - `${jobIdentity(jobInfo)} fused visit prerender of ${url.href} threw: ${(err as Error)?.message}`, + `${jobIdentity(jobInfo)} index visit prerender of ${url.href} threw: ${(err as Error)?.message}`, ); throw err; } - // Route card result when we parsed a card resource. If the composite - // short-circuited (page-unusable/auth), visitResponse.card may be missing. - // In that case, synthesize an error RenderResponse from pageUnusableError - // so the card entry still gets a proper error row rather than being left - // stale. This matches the legacy flow, which always attempts card indexing - // independently of file-level outcomes. + // The prerender-html visit chains off the index visit: the extract's + // resource becomes the FileDef rendering's fileData, the extract's types + // drive the FileDef fitted/embedded renders, and the card meta's types + // drive the card's ancestor renders. Skipped when the index visit left + // the page unusable (mirroring the short-circuit inside a single visit) + // or when there is nothing for it to render. + let htmlResponse: RenderVisitResponse | undefined; + let htmlFileData = indexResponse.fileExtract?.resource + ? { resource: indexResponse.fileExtract.resource, fileDefCodeRef } + : undefined; + let needFileHtml = needFileRender && htmlFileData !== undefined; + if (!indexResponse.pageUnusableError && (needCardRender || needFileHtml)) { + let htmlRenderOptions: RenderRouteOptions = { + fileDefCodeRef, + ...(needCardRender ? { cardRender: true } : {}), + ...(needFileHtml ? { fileRender: true } : {}), + }; + try { + htmlResponse = await prerenderer.prerenderVisit({ + ...visitArgs, + visitType: 'prerender-html', + renderOptions: htmlRenderOptions, + ...(htmlFileData ? { fileData: htmlFileData } : {}), + ...(indexResponse.fileExtract?.types?.length + ? { types: indexResponse.fileExtract.types } + : {}), + ...(indexResponse.card?.types?.length + ? { cardTypes: indexResponse.card.types } + : {}), + }); + } catch (err) { + logWarn( + `${jobIdentity(jobInfo)} prerender-html visit of ${url.href} threw: ${(err as Error)?.message}`, + ); + throw err; + } + } + + let card = mergeCardVisitResults(indexResponse.card, htmlResponse?.card); + let fileRenderResult = mergeFileRenderVisitResults( + indexResponse.fileRender, + htmlResponse?.fileRender, + ); + let pageUnusableError = + indexResponse.pageUnusableError ?? htmlResponse?.pageUnusableError; + let diagnostics = mergeVisitDiagnostics( + flattenPrerenderMeta(indexResponse.meta), + flattenPrerenderMeta(htmlResponse?.meta), + ); + + // Route card result when we parsed a card resource. If the visits + // short-circuited (page-unusable/auth), the card sub-result may be + // missing. In that case, synthesize an error RenderResponse from + // pageUnusableError so the card entry still gets a proper error row + // rather than being left stale — card indexing always runs for a parsed + // card resource, independent of file-level outcomes. if (parsedCardResource) { - let cardResult: NonNullable = - visitResponse.card ?? { - serialized: null, - searchDoc: null, - displayNames: null, - deps: null, - types: null, - isolatedHTML: null, - headHTML: null, - atomHTML: null, - embeddedHTML: null, - fittedHTML: null, - iconHTML: null, - markdown: null, - error: visitResponse.pageUnusableError ?? { - type: 'instance-error', - error: { - message: - 'prerenderVisit returned no card result for a card resource', - status: 500, - additionalErrors: null, - }, + let cardResult: NonNullable = card ?? { + serialized: null, + searchDoc: null, + displayNames: null, + deps: null, + types: null, + isolatedHTML: null, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, + markdown: null, + error: pageUnusableError ?? { + type: 'instance-error', + error: { + message: 'prerenderVisit returned no card result for a card resource', + status: 500, + additionalErrors: null, }, - }; + }, + }; await indexCardWithResult({ path: localPath, lastModified, resourceCreatedAt, resource: parsedCardResource, renderResult: cardResult, - diagnostics: flattenPrerenderMeta(visitResponse.meta), + diagnostics, }); } @@ -264,12 +329,137 @@ export async function visitFileForIndexingFused({ lastModified, resourceCreatedAt, hasModulePrerender: isModule, - extractResult: visitResponse.fileExtract, - renderResult: visitResponse.fileRender, - diagnostics: flattenPrerenderMeta(visitResponse.meta), + extractResult: indexResponse.fileExtract, + renderResult: fileRenderResult, + diagnostics, }); logDebug( - `${jobIdentity(jobInfo)} completed fused visit of file ${url.href} in ${Date.now() - start}ms`, + `${jobIdentity(jobInfo)} completed visit of file ${url.href} in ${Date.now() - start}ms`, ); } + +// Reassemble the two visits' card sub-results into the single RenderResponse +// shape the card indexer consumes: the index visit supplies the search-doc +// side (meta fields + icon), the prerender-html visit supplies the html +// formats + markdown, and the runtime deps are the union of both visits' +// captures — the meta deps cover the search-doc walk, the html visit's +// cover what the format renders pulled in, and both edge sets must land on +// the row for invalidation to reach every dependent. When both visits error, +// they belong to one indexing job — the same index generation — so neither +// supersedes the other: keep the html visit's isolated-render-wrapped, +// card-facing message ("Encountered error rendering HTML for card: …") as the +// primary error and fold the index visit's dependency-error detail into its +// additionalErrors. A card whose computed throws fails serialization (index) +// and template render (html) alike; a card whose adopted module is missing +// carries its dependency chain on the index visit's error, which the html +// render path does not reconstruct. Whichever visit is the sole one to error +// is used as-is. +function mergeCardVisitResults( + index: RenderResponse | undefined, + html: RenderResponse | undefined, +): RenderResponse | undefined { + if (!index && !html) { + return undefined; + } + let error: RenderResponse['error']; + if (index?.error && html?.error) { + error = { + ...html.error, + // Both visits are the same indexing job → the same index generation. + error: mergeErrorsByGeneration(html.error.error, 0, index.error.error, 0), + }; + } else { + error = index?.error ?? html?.error; + } + return { + serialized: index?.serialized ?? null, + searchDoc: index?.searchDoc ?? null, + displayNames: index?.displayNames ?? null, + types: index?.types ?? null, + deps: mergeDeps(index?.deps ?? null, html?.deps ?? null), + ...(index?.diagnostics ? { diagnostics: index.diagnostics } : {}), + iconHTML: index?.iconHTML ?? null, + isolatedHTML: html?.isolatedHTML ?? null, + headHTML: html?.headHTML ?? null, + atomHTML: html?.atomHTML ?? null, + embeddedHTML: html?.embeddedHTML ?? null, + fittedHTML: html?.fittedHTML ?? null, + markdown: html?.markdown ?? null, + ...(error ? { error } : {}), + }; +} + +function mergeFileRenderVisitResults( + index: FileRenderResponse | undefined, + html: FileRenderResponse | undefined, +): FileRenderResponse | undefined { + if (!index && !html) { + return undefined; + } + let error = index?.error ?? html?.error; + return { + iconHTML: index?.iconHTML ?? null, + isolatedHTML: html?.isolatedHTML ?? null, + headHTML: html?.headHTML ?? null, + atomHTML: html?.atomHTML ?? null, + embeddedHTML: html?.embeddedHTML ?? null, + fittedHTML: html?.fittedHTML ?? null, + markdown: html?.markdown ?? null, + ...(error ? { error } : {}), + }; +} + +function mergeDeps(a: string[] | null, b: string[] | null): string[] | null { + if (!a) { + return b; + } + if (!b) { + return a; + } + return [...new Set([...a, ...b])]; +} + +// A row is produced by two sequential visits, each reporting its own +// server-observed timings. Sum the timing fields so the persisted +// diagnostics still answer "how long did this row take to produce"; every +// other field prefers the index visit's value, with the html visit's +// surviving where the index visit has none (e.g. `renderStage` from a +// timed-out format render). The html visit's HTTP correlation id is kept +// under `prerenderHtmlRequestId` so operators can join logs for both. +function mergeVisitDiagnostics( + index: Diagnostics | undefined, + html: Diagnostics | undefined, +): Diagnostics | undefined { + if (!index || !html) { + return index ?? html; + } + let sum = (a?: number, b?: number) => + a === undefined && b === undefined ? undefined : (a ?? 0) + (b ?? 0); + let merged: Diagnostics = { ...html, ...index }; + for (let key of ['launchMs', 'renderElapsedMs', 'totalElapsedMs'] as const) { + let total = sum(index[key], html[key]); + if (total !== undefined) { + merged[key] = total; + } + } + if (index.waits || html.waits) { + let waits: NonNullable = {}; + for (let key of [ + 'semaphoreMs', + 'admissionMs', + 'tabQueueMs', + 'tabStartupMs', + ] as const) { + let total = sum(index.waits?.[key], html.waits?.[key]); + if (total !== undefined) { + waits[key] = total; + } + } + merged.waits = waits; + } + if (html.requestId && html.requestId !== merged.requestId) { + merged.prerenderHtmlRequestId = html.requestId; + } + return merged; +} diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index 1189357170..7430e26c29 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -488,6 +488,11 @@ export interface Diagnostics extends RenderTimeoutDiagnostics, PrerenderMetaDiagnostics { invalidationId?: string; indexedAt?: number; + // A row is produced by two prerender visits (index + prerender-html), + // each its own HTTP request. `requestId` carries the index visit's id; + // this carries the prerender-html visit's so operators can join logs + // for both. Absent for in-process callers and fused single-visit rows. + prerenderHtmlRequestId?: string; // Frontmatter YAML that wouldn't parse during file extraction. The row // still indexes (body-only); this is the only indexed signal that the // file's frontmatter — and anything it declared — was dropped. Merged in @@ -561,16 +566,41 @@ export const VISIT_PASS_ORDER = [ ] as const; export type VisitPass = (typeof VISIT_PASS_ORDER)[number]; +// The consolidated visit splits along the search-doc/HTML seam into two +// visit types: +// +// - 'index' — everything the search index needs: the file extract, the +// card's meta (search doc / serialized / types / display names / deps) +// and the icon render. Never runs the `html` route, never materializes +// a format component via `getComponent`. +// - 'prerender-html' — the `html` route per format (isolated, head, +// atom, fitted, embedded) plus markdown, for both the card and the +// file rendering of a URL. Produces no search-doc data. +// +// A visit with no `visitType` runs the union of both (the fused visit) — +// used by callers that want a complete render in one round-trip (e.g. the +// user-initiated prerender proxy). +export type PrerenderVisitType = 'index' | 'prerender-html'; + export type PrerenderVisitArgs = { affinityType: AffinityType; affinityValue: string; realm: string; url: string; auth: string; + // Selects which half of the bifurcated visit to run — see + // PrerenderVisitType. Absent runs the fused union of both halves. + visitType?: PrerenderVisitType; renderOptions?: RenderRouteOptions; // Inputs required only when the fileRender pass is requested fileData?: FileRenderArgs['fileData']; types?: string[]; + // Ancestor type chain (internalKeyFor form) driving the card's + // fitted/embedded format renders in a 'prerender-html' visit. Supplied + // by callers that already hold the chain (the indexing job passes the + // index visit's `types` through); when absent the visit resolves the + // chain itself via the types route. + cardTypes?: string[]; // Identifies the indexing batch this visit belongs to (CS-10758 step 3). // Required to honor `renderOptions.clearCache: true` on the prerender // server when another batch currently owns the affinity. Visits without @@ -701,6 +731,8 @@ export { coerceErrorMessage, stringifyErrorForLog, sanitizeForJsonb, + mergeErrorDetail, + mergeErrorsByGeneration, ERROR_DOC_MAX_BYTES, ERROR_DOC_MAX_ADDITIONAL_ERRORS, } from './error.ts';