PagePool: materialize the reserved module/command tab; plumb priority through definition lookups

packages/host/app/services/store.ts

@@ -32,6 +32,8 @@ import {
   isSingleCardDocument,
   isLinkableCollectionDocument,
   resolveFileDefCodeRef,
+  X_BOXEL_JOB_PRIORITY_HEADER,
+  userInitiatedPriority,
   Deferred,
   delay,
   mergeRelationships,
@@ -123,6 +125,77 @@ let waiter = buildWaiter('store-service');
 const realmEventsLogger = logger('realm:events');
 const storeLogger = logger('store');
 
+// Companion to `jobIdHeader()` (re-exported from
+// `../lib/prerender-fetch-headers`). Policy is two-state, gated by
+// `__boxelDuringPrerender`, not by the presence of
+// `__boxelJobPriority`:
+//
+// 1. Inside a prerender tab: forward the worker job's priority as-is.
+//    The render-runner injects `__boxelJobPriority` alongside
+//    `__boxelJobId` on each visit — a priority of 0 is meaningful
+//    (the originating job is system-initiated background indexing)
+//    and must be preserved, not upgraded. Sub-`prerenderModule`
+//    calls fired by `_federated-search` for a `lookupDefinition`
+//    cache miss inherit this priority so they don't outrun the
+//    parent. If `__boxelJobPriority` is missing here (older
+//    render-runner build, test fixture, etc.) treat as 0 — the
+//    safe default for prerender-context work.
+//
+// 2. Outside a prerender tab (the host SPA in a real user's browser):
+//    stamp `userInitiatedPriority` (10). User clicks driving a
+//    search are by definition user-initiated work and should outrank
+//    background indexing on the realm-server's PagePool. Without
+//    this, a user search whose definition lookup misses the modules
+//    cache would fire its sub-prerender at priority 0 and queue
+//    behind concurrent indexing fan-out.
+//
+// External (non-host) HTTP callers — anything that doesn't run in
+// the host SPA's JS runtime — bypass this helper entirely and set
+// `X-Boxel-Job-Priority` directly on their request if they care.
+// This helper covers the host SPA only.
+//
+// Both globals are checked with `=== true` / strict-number rather
+// than truthy coercion: `__boxelDuringPrerender` is typed as a
+// boolean and a stray truthy string from a future code path
+// shouldn't silently flip the policy from "user-priority" to
+// "preserve 0."
+// Pure resolver — exported for the unit test in
+// `tests/integration/job-priority-header-test.ts`. See the comment
+// above for the policy rationale; the function is the literal
+// translation of that policy to numbers.
+export function resolveOutboundJobPriority({
+  duringPrerender,
+  jobPriority,
+}: {
+  duringPrerender: unknown;
+  jobPriority: unknown;
+}): number {
+  let valid =
+    typeof jobPriority === 'number' &&
+    Number.isSafeInteger(jobPriority) &&
+    jobPriority >= 0
+      ? jobPriority
+      : undefined;
+  if (duringPrerender === true) {
+    return valid ?? 0;
+  }
+  return valid ?? userInitiatedPriority;
+}
+
+function jobPriorityHeader(): Record<string, string> {
+  let g = globalThis as unknown as {
+    __boxelDuringPrerender?: boolean;
+    __boxelJobPriority?: number;
+  };
+  return {
+    [X_BOXEL_JOB_PRIORITY_HEADER]: String(
+      resolveOutboundJobPriority({
+        duringPrerender: g.__boxelDuringPrerender,
+        jobPriority: g.__boxelJobPriority,
+      }),
+    ),
+  };
+}
 const queryFieldSeedFromSearchSymbol = Symbol.for(
   'cardstack-query-field-seed-from-search',
 );
@@ -835,6 +908,7 @@ export default class StoreService extends Service implements StoreInterface {
           ...duringPrerenderHeaders(),
           ...consumingRealmHeader(),
           ...jobIdHeader(),
+          ...jobPriorityHeader(),
         },
         body: JSON.stringify({ ...query, realms }),
       },
@@ -904,6 +978,7 @@ export default class StoreService extends Service implements StoreInterface {
           ...duringPrerenderHeaders(),
           ...consumingRealmHeader(),
           ...jobIdHeader(),
+          ...jobPriorityHeader(),
         },
         body: JSON.stringify({ ...query, realms }),
       },

packages/host/tests/unit/job-priority-header-test.ts

@@ -0,0 +1,150 @@
+import { module, test } from 'qunit';
+
+import { userInitiatedPriority } from '@cardstack/runtime-common';
+
+import { resolveOutboundJobPriority } from '@cardstack/host/services/store';
+
+// Pure-resolver tests for the policy that decides what
+// `X-Boxel-Job-Priority` value the host SPA stamps on outbound
+// `_federated-search` calls. The function is module-internal logic
+// extracted so its policy can be pinned without acceptance-test
+// scaffolding.
+//
+// Two states gated by `__boxelDuringPrerender`:
+//   - inside prerender → forward (preserve 0)
+//   - outside prerender → user-initiated (10) by default
+module('Unit | job-priority-header | resolveOutboundJobPriority', function () {
+  module('outside a prerender tab (user / API caller)', function () {
+    test('returns userInitiatedPriority when no global is set', function (assert) {
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: undefined,
+          jobPriority: undefined,
+        }),
+        userInitiatedPriority,
+      );
+    });
+
+    test('returns userInitiatedPriority when __boxelDuringPrerender is false', function (assert) {
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: false,
+          jobPriority: undefined,
+        }),
+        userInitiatedPriority,
+      );
+    });
+
+    test('honors an explicit override on __boxelJobPriority', function (assert) {
+      // Batch / scripting tooling running in the host SPA can set the
+      // global before issuing a fetch; outside a prerender tab we still
+      // forward what they set rather than overriding to user priority.
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: undefined,
+          jobPriority: 3,
+        }),
+        3,
+      );
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: false,
+          jobPriority: 0,
+        }),
+        0,
+        'override with 0 is preserved (not coerced to user priority)',
+      );
+    });
+
+    test('rejects a truthy but non-boolean __boxelDuringPrerender — uses strict === true', function (assert) {
+      // If `__boxelDuringPrerender` somehow ended up as a stringy
+      // truthy value (e.g. set by a future code path that didn't
+      // coerce), the policy must NOT silently flip to "forward 0";
+      // a real user-facing fetch would then queue behind background
+      // indexing. The check is `=== true` for exactly this reason.
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: 'yes',
+          jobPriority: undefined,
+        }),
+        userInitiatedPriority,
+      );
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: 1,
+          jobPriority: undefined,
+        }),
+        userInitiatedPriority,
+      );
+    });
+  });
+
+  module('inside a prerender tab', function () {
+    test('forwards an explicit __boxelJobPriority of 10', function (assert) {
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: true,
+          jobPriority: 10,
+        }),
+        10,
+      );
+    });
+
+    test('forwards an explicit __boxelJobPriority of 0 — must NOT upgrade', function (assert) {
+      // System-initiated indexing has priority 0. A
+      // `_federated-search` fired by the card render must preserve
+      // that or its sub-prerenders would outrank the parent job.
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: true,
+          jobPriority: 0,
+        }),
+        0,
+      );
+    });
+
+    test('defaults to 0 when __boxelJobPriority is missing (older render-runner / test fixture)', function (assert) {
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: true,
+          jobPriority: undefined,
+        }),
+        0,
+      );
+    });
+
+    test('rejects malformed __boxelJobPriority values', function (assert) {
+      // Non-number / negative / non-integer values fall through to
+      // the default for the active branch.
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: true,
+          jobPriority: -1,
+        }),
+        0,
+      );
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: true,
+          jobPriority: 1.5,
+        }),
+        0,
+      );
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: true,
+          jobPriority: '10',
+        }),
+        0,
+      );
+      assert.strictEqual(
+        resolveOutboundJobPriority({
+          duringPrerender: false,
+          jobPriority: -1,
+        }),
+        userInitiatedPriority,
+        'malformed value outside prerender → user-initiated default',
+      );
+    });
+  });
+});

packages/realm-server/handlers/handle-search.ts

@@ -22,6 +22,8 @@ import {
 import type { JobScopedSearchCache } from '../job-scoped-search-cache';
 import {
   PRERENDER_JOB_ID_HEADER,
+  PRERENDER_JOB_PRIORITY_HEADER,
+  sanitizeJobPriorityHeader,
   sanitizePrerenderJobId,
 } from '../prerender/prerender-constants';
 
@@ -53,14 +55,27 @@ export default function handleSearch(opts?: {
     }
 
     let cacheOnlyDefinitions = ctxt.get(DURING_PRERENDER_HEADER).length > 0;
-    let searchOpts = cacheOnlyDefinitions
-      ? { cacheOnlyDefinitions: true }
-      : undefined;
+    // The host's `_federated-search` fetch wrapper stamps
+    // `x-boxel-job-priority` while rendering inside a prerender tab.
+    // Threading it into search opts here lets `CachingDefinitionLookup`
+    // sub-prerenders (fired when a `type:` filter misses the modules
+    // cache) inherit the originating job's priority instead of silently
+    // dropping to 0. User / API callers don't stamp the header, so the
+    // value is `null` for live traffic — falls back to priority 0
+    // (system-initiated default), same observable behavior as today.
+    let jobPriority = sanitizeJobPriorityHeader(
+      ctxt.get(PRERENDER_JOB_PRIORITY_HEADER),
+    );
+    let searchOpts: { cacheOnlyDefinitions?: true; priority?: number } = {};
+    if (cacheOnlyDefinitions) searchOpts.cacheOnlyDefinitions = true;
+    if (jobPriority !== null) searchOpts.priority = jobPriority;
+    let normalizedSearchOpts =
+      Object.keys(searchOpts).length > 0 ? searchOpts : undefined;
     let runSearch = () =>
       searchRealms(
         realmList.map((realmURL) => realmByURL.get(realmURL)),
         cardsQuery,
-        searchOpts,
+        normalizedSearchOpts,
       );
 
     // Job-scoped cache. Gated on:

packages/realm-server/prerender/page-pool.ts

@@ -1870,6 +1870,55 @@ export class PagePool {
         let releaseTab = await commandeered.queue.acquire(signal, priority);
         return { entry: commandeered, reused: false, releaseTab, tabStartupMs };
       }
+      // module / command callers must produce the reserved tab the
+      // file-admission cap is supposed to keep room for. The cap
+      // bounds file workload at `affinityTabMax − 1`, leaving global-
+      // pool headroom for a non-file call — but the headroom is only
+      // a reservation, not a spawned tab. Without this synchronous
+      // refill+retry, the call falls through to the busy-tab fallback
+      // below and queues behind the file render that's awaiting this
+      // sub-render: the self-referential prerender deadlock.
+      // `#ensureStandbyPool` respects `#maxPages` via
+      // `#prepareSlotForStandby`, so this can't oversubscribe the
+      // global pool. Note this path only fires under
+      // `entryList.length < #affinityTabMax` — the at-cap case (every
+      // tab held by file renders, no dynamic-expansion budget) still
+      // falls through to busy-tab below. That residual deadlock
+      // requires either operator-side capacity tuning or the high-
+      // priority tier escape hatch beneath this branch.
+      //
+      // We await `#ensureStandbyPool` UNCONDITIONALLY here (not gated
+      // on `current < desired`). Reason: `#currentStandbyCount` =
+      // `#standbys.size + #creatingStandbys`. If the file render that
+      // arrived just before this caller consumed the only standby and
+      // its post-acquire `#kickStandbyRefill` is already creating a
+      // replacement, `creatingStandbys > 0` inflates `current` to
+      // meet `desired` while `#standbys.size` is still 0. A
+      // `current < desired` guard would skip the await; the
+      // subsequent `commandeerDormantTab(standbyOnly:true)` would
+      // then fail to find a real standby and the caller would fall
+      // through to the busy-tab branch — exactly the deadlock this
+      // change is meant to prevent. Two scenarios produce no-op
+      // behavior: (a) the pool is genuinely healthy with
+      // `#standbys.size >= desired` and no refill in flight —
+      // `#ensureStandbyPoolInternal`'s loop returns at line 1266
+      // (`current >= desired`); (b) a refill is in flight —
+      // `#ensureStandbyPool` returns the existing `#ensuringStandbys`
+      // promise via dedup at line 1242 and we wait for it. Both
+      // produce the right shape: no spurious creation when not needed,
+      // wait when needed.
+      if (queue !== 'file' && entryList.length < this.#affinityTabMax) {
+        let startedAt = Date.now();
+        await this.#ensureStandbyPool();
+        tabStartupMs += Date.now() - startedAt;
+        let refilled = this.#commandeerDormantTab(affinityKey, {
+          standbyOnly: true,
+        });
+        if (refilled) {
+          let releaseTab = await refilled.queue.acquire(signal, priority);
+          return { entry: refilled, reused: false, releaseTab, tabStartupMs };
+        }
+      }
       // No orphan, no commandeer-able tab/standby. If we got here
       // through the dynamic-expansion escape hatch, drive an
       // expansion + fresh spawn so the saturated module/command
@@ -1924,6 +1973,18 @@ export class PagePool {
       // #ensureStandbyPool()` in `getPage` made this ordering implicit;
       // now we make it explicit here so brand-new affinities still get
       // a fresh standby in preference to busy-tab queueing.
+      //
+      // The gate here is intentional and asymmetric with the non-file
+      // spawn-branch above. There the deadlock cost of skipping is
+      // unbounded (the caller queues on the very tab whose work it
+      // blocks), so we await unconditionally. Here the cost of
+      // skipping is only a cross-affinity-steal hop — itself a
+      // designed fallback — so paying an extra microtask for a no-op
+      // await is the wrong trade. It also shifts microtask ordering
+      // against a concurrent same-affinity file caller arriving on
+      // an idle tab, which the
+      // `queues same-realm request when tab is transitioning` test in
+      // `prerendering-test.ts` pins.
       if (this.#currentStandbyCount() < this.#desiredStandbyCount()) {
         let startedAt = Date.now();
         await this.#ensureStandbyPool();

packages/realm-server/prerender/prerender-constants.ts

@@ -42,6 +42,16 @@ export function sanitizePrerenderRequestId(
 // imports keep working unchanged.
 export { X_BOXEL_JOB_ID_HEADER as PRERENDER_JOB_ID_HEADER } from '@cardstack/runtime-common';
 
+// Worker-job priority of the request that triggered this prerender.
+// Producer side stamps this header so any sub-`prerenderModule` the
+// host fires during render inherits the originating priority instead
+// of silently dropping to 0 — see prerender-headers.ts for the full
+// chain rationale.
+export {
+  X_BOXEL_JOB_PRIORITY_HEADER as PRERENDER_JOB_PRIORITY_HEADER,
+  sanitizeJobPriorityHeader,
+} from '@cardstack/runtime-common';
+
 // Stamped on the host's outbound _federated-search / _search calls
 // when the host SPA detects it's running inside a prerender tab. The
 // prerender server signals "you are in a prerender" by injecting