refactor: split the prerender visit into index and prerender-html passes

packages/base/card-serialization.ts

@@ -66,6 +66,11 @@ export interface JSONAPISingleResourceDocument {
 
 export interface SerializeOpts {
   includeComputeds?: boolean;
+  // Include link fields the card does not actually have. By default
+  // serialization keeps only the relationships present in the card's data — a
+  // set target or an authored empty `{ self: null }` — plus contained fields
+  // (always present); set this to serialize every declared relationship,
+  // including never-authored ones (as `{ self: null }`).
   includeUnrenderedFields?: boolean;
   useAbsoluteURL?: boolean;
   omitFields?: [typeof BaseDef];

packages/base/field-support.ts

@@ -194,7 +194,15 @@ export function getter<CardT extends BaseDefConstructor>(
       return deserialized.get(field.name);
     }
     let value = field.emptyValue(instance);
-    deserialized.set(field.name, value);
+    // Don't persist a nullish empty value (an unset `linksTo`): writing it would
+    // fabricate a bucket entry for a link the caller merely read, making an
+    // unset link indistinguishable from an authored `{ self: null }` when the
+    // card serializes. Recomputing the nullish empty on the next read is free;
+    // identity only matters for the mutable empties (arrays / composite fields),
+    // which are non-null and still persist for reactivity.
+    if (value != null) {
+      deserialized.set(field.name, value);
+    }
     return value;
   }
 }
@@ -350,13 +358,14 @@ export function getFieldOverrides<T extends BaseDef>(
 // per serialize, per searchDoc, per getDeps/findInstances recursion — and
 // `computeFields` walks the prototype chain and resolves every field on each
 // call. The result is determined by the prototype chain plus, for an instance,
-// its polymorphic field overrides (and, with `usedLinksToFieldsOnly`, its
-// populated field set). A read-only render only ever GROWS those, so their
-// sizes are a sound validity token: an unchanged token means an identical
-// result. Gated to the render context so the live app — where instances mutate
-// freely (same-size override swaps, field clears) — is never served a memoized
-// map. Keyed on the instance/class via a WeakMap so entries fall away with
-// their subjects; the token guards reuse across passes.
+// its polymorphic field overrides and — under `usedLinksToFieldsOnly` — the set
+// of links it has (authored or set): a link joins the map as its data-bucket
+// entry appears. A read-only render only ever GROWS both, so their sizes are a
+// sound validity token: an unchanged token means an identical result. Gated to
+// the render context so the live app — where
+// instances mutate freely (same-size override swaps, field clears) — is never
+// served a memoized map. Keyed on the instance/class via a WeakMap so entries
+// fall away with their subjects; the token guards reuse across passes.
 const renderFieldsCache = new WeakMap<
   object,
   Map<
@@ -380,8 +389,14 @@ function renderFieldsCacheToken(
   }
   let overrideSize = fieldOverrides.get(subject as BaseDef)?.size ?? 0;
   if (!usedLinksToFieldsOnly) {
+    // The full field map lists every declared field, so it depends only on the
+    // prototype chain and polymorphic overrides — not on what the instance set.
     return `o${overrideSize}`;
   }
+  // A link is kept only when the card actually has it (authored or set), so
+  // this map — unlike the full map — also depends on the instance's data. The
+  // data bucket only grows within a render pass, so its size is a sound
+  // validity token for that dependency.
   let usedSize = deserializedData.get(subject as BaseDef)?.size ?? 0;
   return `o${overrideSize}:u${usedSize}`;
 }
@@ -432,11 +447,11 @@ function computeFields(
   opts?: { usedLinksToFieldsOnly?: boolean; includeComputeds?: boolean },
 ): { [fieldName: string]: Field<BaseDefConstructor> } {
   let obj: object | null;
-  let usedFields: string[] = [];
+  let instance: BaseDef | undefined;
   if (isCardOrField(cardInstanceOrClass)) {
     // this is a card instance
+    instance = cardInstanceOrClass;
     obj = Reflect.getPrototypeOf(cardInstanceOrClass);
-    usedFields = getUsedFields(cardInstanceOrClass);
   } else {
     // this is a card class
     obj = (cardInstanceOrClass as typeof BaseDef).prototype;
@@ -460,9 +475,15 @@ function computeFields(
         maybeField.computeVia ||
         !['contains', 'containsMany'].includes(maybeField.fieldType)
       ) {
+        // `usedLinksToFieldsOnly` keeps only the link fields the card actually
+        // has — an authored empty (`{ self: null }`) or a set target — and drops
+        // never-authored links. This is a property of the card's data, not of
+        // searchability (searchable annotations drive the search doc, not this
+        // serialization). Callers wanting every declared link pass
+        // `includeUnrenderedFields`; contained fields are always kept.
         if (
           opts?.usedLinksToFieldsOnly &&
-          !usedFields.includes(maybeFieldName) &&
+          !isFieldUsed(instance, maybeFieldName) &&
           !['contains', 'containsMany'].includes(maybeField.fieldType)
         ) {
           return [];
@@ -483,10 +504,35 @@ function computeFields(
   return fields;
 }
 
-function getUsedFields(instance: BaseDef): string[] {
-  // getDataBucket always returns a Map (it creates one if absent), so the spread
-  // is safe without optional chaining.
-  return [...getDataBucket(instance).keys()];
+// Whether a link field is "used" for `usedLinksToFieldsOnly` serialization: the
+// card actually has the relationship. A link enters the data bucket only when
+// it was authored (deserialized from the source — a set target, or an empty
+// `{ self: null }` that lands as a `null` entry) or set on the instance; a
+// never-authored `linksTo` has no bucket entry, because the getter skips
+// persisting its nullish empty value, so a mere render can't mark it "used".
+// A `linksToMany` getter must persist its (mutable) backing array even when
+// empty, so an empty array is NOT a "had" relationship — reading an unset
+// plural link would otherwise fabricate one. This is pure card data,
+// independent of searchability. Contained fields never reach here (always kept).
+function isFieldUsed(
+  instance: BaseDef | undefined,
+  fieldName: string,
+): boolean {
+  if (!instance) {
+    return false;
+  }
+  let bucket = getDataBucket(instance);
+  if (!bucket.has(fieldName)) {
+    return false;
+  }
+  let value = bucket.get(fieldName);
+  // An empty backing array is a `linksToMany` the card doesn't actually have
+  // (never authored, or materialized empty on read); `null` is an authored
+  // empty `linksTo` and is kept.
+  if (Array.isArray(value)) {
+    return value.length > 0;
+  }
+  return true;
 }
 
 export function isArrayOfCardOrField(
@@ -807,7 +853,8 @@ export interface BrokenLinkFinding {
 // planted a sentinel there), and an unmaterialized field holds neither a broken
 // link nor a nested card — so skipping absent fields loses nothing and avoids
 // the getter's side effect of initializing them with `emptyValue` (which would
-// pollute `getUsedFields` / serialization). Relationship state is then read
+// pollute the data-bucket-driven `isFieldUsed` signal / serialization).
+// Relationship state is then read
 // through `getRelationshipMembershipState`, which never triggers `lazilyLoadLink`, so a
 // recursed value surfaces only states that genuinely failed during this render.
 // `'present'`, `'not-loaded'`, and `'not-set'` are not terminal failures; a
@@ -859,19 +906,6 @@ export function getBrokenLinks(
       );
       for (let entry of membership ?? []) {
         if (entry.kind === 'error' || entry.kind === 'not-found') {
-          // DIAGNOSTIC LOGGING (CS-11221) — remove after CI passes. Read
-          // only fields that won't initialize bucket entries via the
-          // field getter (constructor name + the entry's own reference);
-          // reading `instance.id` here would write the `id` field's
-          // emptyValue into the bucket and violate the pure-read contract
-          // the surrounding `getBrokenLinks` upholds.
-          console.error('[CS-11221 DIAG] getBrokenLinks finding', {
-            ownerType: instance?.constructor?.name,
-            fieldName,
-            fieldType: field.fieldType,
-            kind: entry.kind,
-            reference: entry.reference,
-          });
           findings.push({
             fieldName,
             kind: entry.kind,

packages/base/searchable.ts

@@ -6,6 +6,7 @@ import {
   primitive,
   relativeTo,
   routesForField,
+  type SerializedError,
 } from '@cardstack/runtime-common';
 import {
   getDataBucket,
@@ -15,6 +16,8 @@ import {
   isNonPresentLink,
   isNotLoadedValue,
   peekAtField,
+  type LinkErrorValue,
+  type LinkNotFoundValue,
 } from './field-support';
 import {
   createFromSerialized,
@@ -81,36 +84,73 @@ function seedSearchableRoutes(cardClass: typeof BaseDef): string[] {
   return routes;
 }
 
+// Outcome of loading a searchable link target: the deserialized card, or the
+// terminal sentinel a failed load resolves to. A broken searchable link is
+// captured as `{ id }` in the doc AND recorded so the caller can plant the
+// sentinel on the owner's field — the diagnostic `getBrokenLinks` reads.
+type SearchableTargetResult =
+  | { status: 'loaded'; card: CardDef }
+  | { status: 'broken'; sentinel: LinkErrorValue | LinkNotFoundValue };
+
+// Build the terminal sentinel a broken link resolves to, mirroring the shape
+// `lazilyLoadLink` plants: HTTP 404 → `link-not-found`, anything else →
+// `link-error`. `reference` is the resolved (absolute) target url.
+function brokenLinkSentinel(
+  reference: string,
+  err: unknown,
+): LinkErrorValue | LinkNotFoundValue {
+  let status = Number((err as { status?: unknown })?.status);
+  let message =
+    (err as { message?: unknown })?.message != null
+      ? String((err as { message?: unknown }).message)
+      : `unable to load ${reference}`;
+  let isMissing =
+    status === 404 || /not found/i.test(message) || /missing/i.test(message);
+  let errorDoc: SerializedError = {
+    status: Number.isFinite(status) ? status : isMissing ? 404 : 500,
+    title: isMissing ? 'Link Not Found' : 'Link Error',
+    message,
+    additionalErrors: null,
+  };
+  return isMissing
+    ? { type: 'link-not-found', reference, errorDoc }
+    : { type: 'link-error', reference, errorDoc };
+}
+
 // Targeted load of a link target by reference: reuse a fully-deserialized
 // resident instance when present, else load + deserialize the document (the
 // same load path the lazy link getter uses). The store load is not itself
 // dependency-tracked; callers record each expanded target in the `dependencies`
-// set instead. Returns undefined if the target errors.
+// set instead. A missing / broken target resolves to a terminal sentinel; the
+// store may surface that either as a returned `CardError` or a thrown rejection
+// (e.g. a 404 / invalid-URL on the load path), so both are captured.
 async function loadSearchableTarget(
   store: CardStore,
   reference: string,
-): Promise<CardDef | undefined> {
+): Promise<SearchableTargetResult> {
   let resident = store.getCard(reference);
   if (resident && (resident as any)[isSavedInstance] === true) {
-    return resident;
+    return { status: 'loaded', card: resident };
   }
-  // A missing / broken target (or one that can't be loaded) degrades to
-  // `{ id }` upstream. The
-  // store may surface that either as a returned `CardError` or a thrown
-  // rejection (e.g. a 404 / invalid-URL on the load path), so guard both.
   try {
     let cardDoc = await store.loadCardDocument(reference);
     if (isCardError(cardDoc)) {
-      return undefined;
+      return {
+        status: 'broken',
+        sentinel: brokenLinkSentinel(reference, cardDoc),
+      };
     }
-    return (await createFromSerialized(
-      cardDoc.data,
-      cardDoc,
-      cardDoc.data.id!,
-      { store },
-    )) as CardDef;
-  } catch {
-    return undefined;
+    return {
+      status: 'loaded',
+      card: (await createFromSerialized(
+        cardDoc.data,
+        cardDoc,
+        cardDoc.data.id!,
+        { store },
+      )) as CardDef,
+    };
+  } catch (err) {
+    return { status: 'broken', sentinel: brokenLinkSentinel(reference, err) };
   }
 }
 
@@ -227,6 +267,7 @@ async function searchableQueryableValue(
         entries.push([
           fieldName,
           await searchableLink(
+            value,
             field!,
             rawValue,
             matched,
@@ -264,6 +305,7 @@ async function searchableQueryableValue(
 // broken / cannot be loaded); the expanded declared-type target when a route names
 // it. Mirrors `LinksTo.queryableValue` + the `{ id }` sentinel handling.
 async function searchableLink(
+  owner: any,
   field: Field<BaseDefConstructor>,
   rawValue: any,
   matched: boolean,
@@ -277,8 +319,17 @@ async function searchableLink(
     return null;
   }
   // A broken / not-found link can't be expanded — keep its reference as `{ id }`.
+  // A searchable target stays a dependency even when broken: recording it
+  // reindexes the card (clearing the `{ id }` / brokenLinks diagnostic) once the
+  // target becomes reachable, mirroring the successful-expansion dep below. This
+  // branch also covers a prior settle pass having planted the sentinel, so the
+  // authoritative generation reaches it here instead of the load branch below.
   if (isLinkError(rawValue) || isLinkNotFound(rawValue)) {
-    return { id: makeAbsoluteURL(rawValue.reference) };
+    let reference = makeAbsoluteURL(rawValue.reference);
+    if (matched) {
+      dependencies.add(reference);
+    }
+    return { id: reference };
   }
   if (!matched) {
     return {
@@ -296,11 +347,20 @@ async function searchableLink(
     // getter. The store can't `toURL` a relative string, which would otherwise
     // degrade an expandable searchable link to `{ id }`.
     let resolvedRef = makeAbsoluteURL(rawValue.reference);
-    let loaded = await loadSearchableTarget(store, resolvedRef);
-    if (loaded == null) {
+    let result = await loadSearchableTarget(store, resolvedRef);
+    if (result.status === 'broken') {
+      // Plant the terminal sentinel on the owner's field so `getBrokenLinks`
+      // (which reads terminal sentinels from the data bucket) records this
+      // broken searchable link. This mirrors the sentinel `lazilyLoadLink`
+      // plants during a template render; search-doc generation plants its own
+      // because it drives the link load directly rather than through a template.
+      getDataBucket(owner).set(field.name, result.sentinel);
+      // A broken searchable target is still a dependency — see the sentinel
+      // branch above.
+      dependencies.add(resolvedRef);
       return { id: resolvedRef };
     }
-    target = loaded;
+    target = result.card;
   }
   // The expanded target's data is now in the doc, so it is a dependency of the
   // indexed card.
@@ -318,7 +378,9 @@ async function searchableLink(
 }
 
 // A `linksToMany` value: per-slot `{ id }` / expansion, with the absolute-URL
-// id normalization `LinksToMany.queryableValue` applies.
+// id normalization `LinksToMany.queryableValue` applies. A broken searchable
+// slot is planted back into the backing array (which lives in the owner's data
+// bucket) so `getBrokenLinks` records it — see `searchableLink`.
 async function searchableLinksToMany(
   field: Field<BaseDefConstructor>,
   rawValue: any,
@@ -335,12 +397,18 @@ async function searchableLinksToMany(
     return null;
   }
   let out: any[] = [];
-  for (let item of rawArrayValues(rawValue)) {
+  let backing = rawArrayValues(rawValue);
+  for (let [index, item] of backing.entries()) {
     if (item == null) {
       continue;
     }
     if (isLinkError(item) || isLinkNotFound(item)) {
-      out.push({ id: makeAbsoluteURL(item.reference) });
+      let reference = makeAbsoluteURL(item.reference);
+      // A broken searchable element stays a dependency — see `searchableLink`.
+      if (matched) {
+        dependencies.add(reference);
+      }
+      out.push({ id: reference });
       continue;
     }
     if (!matched) {
@@ -355,12 +423,19 @@ async function searchableLinksToMany(
     if (isNotLoadedValue(item)) {
       // Resolve a relative reference before the load — see `searchableLink`.
       let resolvedRef = makeAbsoluteURL(item.reference);
-      let loaded = await loadSearchableTarget(store, resolvedRef);
-      if (loaded == null) {
+      let result = await loadSearchableTarget(store, resolvedRef);
+      if (result.status === 'broken') {
+        // Plant the sentinel into the failed slot so `getBrokenLinks` records
+        // this broken searchable element (see `searchableLink`). Assign through
+        // the proxy so the mutation reaches the data bucket the diagnostic
+        // reads.
+        rawValue[index] = result.sentinel;
+        // A broken searchable element stays a dependency — see `searchableLink`.
+        dependencies.add(resolvedRef);
         out.push({ id: resolvedRef });
         continue;
       }
-      target = loaded;
+      target = result.card;
     }
     // The expanded target's data is now in the doc, so it is a dependency of
     // the indexed card.

packages/boxel-cli/tests/commands/ingest-card.test.ts

@@ -15,7 +15,6 @@ describe('ingest-card helpers', () => {
             'bottles.0': { links: { self: '../WineBottle/a' } },
             'bottles.1': { links: { self: '../WineBottle/b' } },
             label: { links: { self: '../Image/x' } },
-            'cardInfo.theme': { links: { self: null } },
           },
         },
       });