From cbd45c9cba12f9f2efa8cbc926d6bdd3f331b1e5 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 9 Oct 2026 10:00:25 +0200 Subject: [PATCH 1/3] Render only changed cards when republishing a realm The publish copy now keeps file mtimes, so a republish indexes only the files that changed and their dependents. Every card still renders again on a first publish, when the published realm.json changed, and when the realm was last published before this server process started (a deploy). Files written in the ambiguous second of the previous publish copy get a fresh mtime, and a publish's index pass no longer joins an in-flight pass that may have read mtimes before the swap. Co-Authored-By: Claude Opus 5.5 --- .../handlers/handle-publish-realm.ts | 139 +++++++++++++-- .../tests/publish-unpublish-realm-test.ts | 158 ++++++++++++++++++ packages/realm-server/tests/queue-test.ts | 62 +++++++ packages/runtime-common/tasks/indexer.ts | 10 +- 4 files changed, 355 insertions(+), 14 deletions(-) diff --git a/packages/realm-server/handlers/handle-publish-realm.ts b/packages/realm-server/handlers/handle-publish-realm.ts index 15616b97aa3..fe13562fd19 100644 --- a/packages/realm-server/handlers/handle-publish-realm.ts +++ b/packages/realm-server/handlers/handle-publish-realm.ts @@ -24,7 +24,18 @@ import fsExtra from 'fs-extra'; // Async fs ops only: the publish handler runs these inside the request, and a // synchronous copy/move of a whole realm directory would freeze the Node event // loop, stalling every other concurrent request until it finished. -const { copy, readJson, writeJson, remove, pathExists, move } = fsExtra; +const { + copy, + readJson, + writeJson, + remove, + pathExists, + move, + readFile, + readdir, + stat, + utimes, +} = fsExtra; import { fetchRequestFromContext, @@ -278,6 +289,66 @@ async function dropRealmPolicyPointer(realmPath: string): Promise { await writeJson(realmJsonPath, realmConfigDoc, { spaces: 2 }); } +// A deploy restarts this process. A realm last published before this moment +// was rendered by host and base code that may have changed since, so its +// republish renders every card again. A restart without a deploy costs one +// such full render too. +const PROCESS_STARTED_AT = Date.now() - process.uptime() * 1000; + +async function readRealmConfig(realmPath: string): Promise { + let realmJsonPath = join(realmPath, 'realm.json'); + return (await pathExists(realmJsonPath)) + ? await readFile(realmJsonPath) + : undefined; +} + +function sameRealmConfig(a: Buffer | undefined, b: Buffer | undefined) { + return a === undefined || b === undefined ? a === b : a.equals(b); +} + +// The copy keeps every file's mtime, so the index skips the files that did +// not change since the last publish. The publish then rewrites the copied +// realm.json (dropRealmPolicyPointer, ensureRealmIndexBoilerplateOptIn), and +// each write stamps it with the current time. Give it the source file's mtime +// back, so an unchanged realm.json also reads as unchanged. +async function restoreRealmConfigMtime( + sourceRealmPath: string, + publishedRealmPath: string, +): Promise { + let sourceRealmJsonPath = join(sourceRealmPath, 'realm.json'); + let publishedRealmJsonPath = join(publishedRealmPath, 'realm.json'); + if ( + !(await pathExists(sourceRealmJsonPath)) || + !(await pathExists(publishedRealmJsonPath)) + ) { + return; + } + let { atime, mtime } = await stat(sourceRealmJsonPath); + await utimes(publishedRealmJsonPath, atime, mtime); +} + +// The index keeps file mtimes in whole seconds. A file written in the second +// the previous publish copied it, and written again later in that same +// second, keeps its mtime in the index and reads as unchanged. Stamp each +// file modified at or after the second the previous publish started its copy +// with the current time, so the index renders it again. Such a file renders +// once more at the next republish, since its index row then holds the stamp +// and not the source mtime. +async function touchFilesModifiedSince( + dir: string, + since: number, + now = new Date(), +): Promise { + for (let entry of await readdir(dir, { withFileTypes: true })) { + let path = join(dir, entry.name); + if (entry.isDirectory()) { + await touchFilesModifiedSince(path, since, now); + } else if (entry.isFile() && (await stat(path)).mtimeMs >= since) { + await utimes(path, now, now); + } + } +} + export default function handlePublishRealm({ dbAdapter, definitionLookup, @@ -484,12 +555,16 @@ export default function handlePublishRealm({ // mounts the (re-)published realm on its first request. The // response is 202 Accepted with status:'pending'; the client polls // //_readiness-check to learn when it's ready. - let { lastPublishedAt, publishedRealmId, isNewRealm } = + let { lastPublishedAt, publishedRealmId, isNewRealm, renderEveryCard } = await dbAdapter.withWriteLock(publishedRealmURL, async () => { let existingRows = (await query(dbAdapter, [ - `SELECT disk_id, owner_username FROM realm_registry WHERE kind = 'published' AND url =`, + `SELECT disk_id, owner_username, last_published_at FROM realm_registry WHERE kind = 'published' AND url =`, param(publishedRealmURL), - ])) as { disk_id: string; owner_username: string }[]; + ])) as { + disk_id: string; + owner_username: string; + last_published_at: string | number | null; + }[]; let isNewRealm = existingRows.length === 0; let publishedRealmId: string; @@ -533,6 +608,12 @@ export default function handlePublishRealm({ let sourceRealmPath = sourceRealm.dir; let publishedDir = join(realmsRootPath, PUBLISHED_DIRECTORY_NAME); let publishedRealmPath = join(publishedDir, publishedRealmId); + let previousRealmConfig = isNewRealm + ? undefined + : await readRealmConfig(publishedRealmPath); + let previousPublishedAt = Number( + existingRows[0]?.last_published_at ?? 0, + ); // Copy source to a temporary directory first, then swap it into // place so that a failed copy doesn't destroy the existing @@ -547,7 +628,19 @@ export default function handlePublishRealm({ let backupPath = `${publishedRealmPath}.backup`; await remove(tempCopyPath); await remove(backupPath); - await copy(sourceRealmPath, tempCopyPath); + // Recorded before the copy, so the next republish knows from which + // second on a copied file's mtime is ambiguous (see + // touchFilesModifiedSince). + let publishStartedAt = Date.now(); + await copy(sourceRealmPath, tempCopyPath, { + preserveTimestamps: true, + }); + if (!isNewRealm) { + await touchFilesModifiedSince( + tempCopyPath, + Math.floor(previousPublishedAt / 1000) * 1000, + ); + } try { await dropRealmPolicyPointer(tempCopyPath); if (await pathExists(publishedRealmPath)) { @@ -601,6 +694,23 @@ export default function handlePublishRealm({ // card (/realm.json) on disk before the reindex below picks // it up. await ensureRealmIndexBoilerplateOptIn(publishedRealmPath); + await restoreRealmConfigMtime(sourceRealmPath, publishedRealmPath); + + // A republish renders again only the files that changed since the + // last publish, and the files that depend on them. The published + // URL is the same as last time, so the rows of unchanged cards + // already hold it. Render every card when that is not enough: a + // new realm has no rows; realm.json feeds realm info (og:title and + // more) into cards that do not depend on it in the index; and code + // deployed since the last publish can render the same files + // differently. + let renderEveryCard = + isNewRealm || + previousPublishedAt < PROCESS_STARTED_AT || + !sameRealmConfig( + previousRealmConfig, + await readRealmConfig(publishedRealmPath), + ); // Clear stale modules cache for the published realm (including // error entries from a previous publish) before the reindex's @@ -617,7 +727,7 @@ export default function handlePublishRealm({ // their own next invalidation arrived. await definitionLookup.clearRealmDefinitions(publishedRealmURL); - let lastPublishedAt = Date.now().toString(); + let lastPublishedAt = publishStartedAt.toString(); try { await upsertPublishedRealmInRegistry(dbAdapter, { publishedRealmURL, @@ -678,8 +788,8 @@ export default function handlePublishRealm({ // lazy-mounts it and start()'s from-scratch pass coalesces with // this job. For a republish already mounted here, the post-lock // fullIndex below tracks completion for readiness. clearLastModified - // forces every row to re-render even though file copies preserve - // mtimes. + // forces every row to re-render even where its mtime did not + // change. await enqueueReindexRealmJob( publishedRealmURL, realmUsername, @@ -689,10 +799,15 @@ export default function handlePublishRealm({ // This publish is blocked on the published realm's HTML (readiness // gates on it), so the prerender-html job this pass spawns runs // co-equal with indexing rather than one tier below. - { clearLastModified: true, awaitedByPublish: true }, + { clearLastModified: renderEveryCard, awaitedByPublish: true }, ); - return { lastPublishedAt, publishedRealmId, isNewRealm }; + return { + lastPublishedAt, + publishedRealmId, + isNewRealm, + renderEveryCard, + }; }); // Mount the published realm on this instance so it is served as soon as @@ -723,7 +838,7 @@ export default function handlePublishRealm({ // Republish: the realm already has index rows, so start() does NOT // re-index them and #startedUp resolves without reflecting the swapped // files — readinessCheck must instead wait on indexing(). Register a - // tracked clearLastModified reindex SYNCHRONOUSLY (before the 202) so + // tracked reindex SYNCHRONOUSLY (before the 202) so // indexing() reflects it and readiness can't report ready before the // reindex lands. Get the mounted realm, or mount it first when this // instance is cold (e.g. after a restart, or the publish landed on an @@ -738,7 +853,7 @@ export default function handlePublishRealm({ if (publishedRealm) { void publishedRealm .fullIndex(userInitiatedPriority, { - clearLastModified: true, + clearLastModified: renderEveryCard, // Republish is awaiting this HTML for readiness, so its render // runs co-equal with indexing (see prerenderHtmlPriority). awaitedByPublish: true, diff --git a/packages/realm-server/tests/publish-unpublish-realm-test.ts b/packages/realm-server/tests/publish-unpublish-realm-test.ts index 15217e4981d..48d7c82b459 100644 --- a/packages/realm-server/tests/publish-unpublish-realm-test.ts +++ b/packages/realm-server/tests/publish-unpublish-realm-test.ts @@ -12,6 +12,7 @@ const { readJsonSync, writeJsonSync, removeSync, + utimesSync, } = fsExtra; import { basename, join } from 'path'; import type { RealmHttpServer as Server } from '../server.ts'; @@ -1507,6 +1508,163 @@ module(basename(import.meta.filename), function () { ); }); + module('republishing a realm', function (hooks) { + let publishedRealmURL = 'http://testuser.localhost:4445/test-realm/'; + let sourceRealmFsPath: string; + let publish: () => Promise; + let readCardRow: ( + name: string, + ) => Promise<{ indexedAt: string; renderedAt: string; head: string }>; + let cardJson = (name: string) => ({ + data: { + type: 'card', + attributes: { cardInfo: { name } }, + meta: { + adoptsFrom: { + module: '@cardstack/base/card-api', + name: 'CardDef', + }, + }, + }, + }); + // Card files written well before the publish, so their mtimes are + // not in the second the publish starts its copy (see + // touchFilesModifiedSince in the publish handler). + let writeCard = (fileName: string, name: string) => { + let path = join(sourceRealmFsPath, fileName); + writeJsonSync(path, cardJson(name)); + let past = new Date(Date.now() - 60_000); + utimesSync(path, past, past); + }; + + hooks.beforeEach(async function () { + let sourceRealmURL = new URL(sourceRealmUrlString); + sourceRealmFsPath = join( + dir.name, + 'realm_server_3', + ...sourceRealmURL.pathname.split('/').filter(Boolean), + ); + let auth = `Bearer ${createRealmServerJWT( + { user: ownerUserId, sessionRoom: 'session-room-test' }, + realmSecretSeed, + )}`; + publish = async () => { + let response = await request + .post('/_publish-realm') + .set('Accept', 'application/vnd.api+json') + .set('Content-Type', 'application/json') + .set('Authorization', auth) + .send( + JSON.stringify({ + sourceRealmURL: sourceRealmUrlString, + publishedRealmURL, + }), + ); + if (response.status !== 202) { + throw new Error(`publish failed with status ${response.status}`); + } + await testRealmServer.testingOnlyReconcile(); + await waitUntil( + async () => + ( + await request + .get( + `${new URL(publishedRealmURL).pathname}_readiness-check?awaitPrerenderHtml=true`, + ) + .set('Host', new URL(publishedRealmURL).host) + .set('Accept', 'application/vnd.api+json') + ).status === 200, + { + timeout: 120_000, + interval: 500, + timeoutMessage: + 'published realm never passed its readiness check', + }, + ); + }; + readCardRow = async (name: string) => { + let [row] = (await dbAdapter.execute( + `SELECT i.indexed_at, h.rendered_at, h.head_html + FROM boxel_index i + JOIN prerendered_html h + ON h.url = i.url AND h.realm_url = i.realm_url AND h.type = i.type + WHERE i.realm_url = $1 + AND i.type = 'instance' + AND i.url = $2`, + { bind: [publishedRealmURL, `${publishedRealmURL}${name}.json`] }, + )) as { + indexed_at: string; + rendered_at: string; + head_html: string | null; + }[]; + if (!row) { + throw new Error( + `no index row for ${name} in the published realm`, + ); + } + return { + indexedAt: String(row.indexed_at), + renderedAt: String(row.rendered_at), + head: row.head_html ?? '', + }; + }; + }); + + test('it renders again only the cards that changed', async function (assert) { + assert.timeout(300_000); + writeCard('changed-card.json', 'Changed card, first version'); + writeCard('unchanged-card.json', 'Unchanged card'); + await publish(); + let unchangedBefore = await readCardRow('unchanged-card'); + + writeCard('changed-card.json', 'Changed card, second version'); + await publish(); + + let changedAfter = await readCardRow('changed-card'); + let unchangedAfter = await readCardRow('unchanged-card'); + assert.true( + changedAfter.head.includes('Changed card, second version'), + 'the changed card is rendered again', + ); + assert.strictEqual( + unchangedAfter.indexedAt, + unchangedBefore.indexedAt, + 'the unchanged card is not indexed again', + ); + assert.strictEqual( + unchangedAfter.renderedAt, + unchangedBefore.renderedAt, + 'the unchanged card is not rendered again', + ); + assert.true( + unchangedAfter.head.includes( + `property="og:url" content="${publishedRealmURL}unchanged-card"`, + ), + `the unchanged card keeps the published URL in its head: ${unchangedAfter.head}`, + ); + }); + + test('it renders every card again when realm.json changed', async function (assert) { + assert.timeout(300_000); + writeCard('unchanged-card.json', 'Unchanged card'); + await publish(); + let unchangedBefore = await readCardRow('unchanged-card'); + + let realmJsonPath = join(sourceRealmFsPath, 'realm.json'); + let realmConfig = readJsonSync(realmJsonPath); + realmConfig.data.attributes.name = 'Renamed Realm'; + writeJsonSync(realmJsonPath, realmConfig); + await publish(); + + let unchangedAfter = await readCardRow('unchanged-card'); + assert.notStrictEqual( + unchangedAfter.renderedAt, + unchangedBefore.renderedAt, + 'the unchanged card is rendered again', + ); + }); + }); + // The in-process readiness gates only see indexing this instance // started. In a deployment with several realm-server replicas behind one // load balancer, a readiness poll can be routed to a replica that did diff --git a/packages/realm-server/tests/queue-test.ts b/packages/realm-server/tests/queue-test.ts index 019a2ad9bf5..b57d5f5d7f3 100644 --- a/packages/realm-server/tests/queue-test.ts +++ b/packages/realm-server/tests/queue-test.ts @@ -894,6 +894,68 @@ module(basename(import.meta.filename), function () { } }); + test('from-scratch dedup: a realm publish does NOT attach to an in-flight from-scratch', async function (assert) { + // A realm publish enqueues its pass after it swaps in new files. An + // already-running from-scratch may have read the file mtimes before + // that swap, so joining it would let the publish observe a successful + // job that never indexed the new files. + await runner.destroy(); + let realmURL = 'http://example.com/from-scratch-realm-publish/'; + let started = new Deferred(); + let release = new Deferred(); + + let worker = new PgQueueRunner({ + adapter, + workerId: 'from-scratch-realm-publish-worker', + }); + worker.register('from-scratch-index', async () => { + started.fulfill(); + await release.promise; + return { + invalidations: [], + ignoreData: {}, + stats: { + instancesIndexed: 0, + filesIndexed: 0, + instanceErrors: 0, + fileErrors: 0, + totalIndexEntries: 0, + }, + }; + }); + + try { + await worker.start(); + + let first = await publishFromScratchIndexJob({ + priority: 0, + args: { realmURL, realmUsername: 'owner' }, + }); + await started.promise; + + let second = await publishFromScratchIndexJob({ + priority: userInitiatedPriority, + args: { + realmURL, + realmUsername: 'owner', + awaitedByPublish: true, + }, + }); + + assert.notStrictEqual( + first.id, + second.id, + 'realm publish does not attach to the in-flight job', + ); + + release.fulfill(); + await Promise.all([first.done, second.done]); + } finally { + release.fulfill(); + await worker.destroy(); + } + }); + test('from-scratch does not coalesce onto pending incremental in same group', async function (assert) { await runner.destroy(); diff --git a/packages/runtime-common/tasks/indexer.ts b/packages/runtime-common/tasks/indexer.ts index 4d0fecea1eb..4772ba280c1 100644 --- a/packages/runtime-common/tasks/indexer.ts +++ b/packages/runtime-common/tasks/indexer.ts @@ -513,8 +513,14 @@ function chooseFromScratchCoalesceDecision( // change. An already-running from-scratch read its mtimes snapshot // before that clear, so attaching this publish to it would let the // caller observe a successful job that did NOT actually re-render - // the swapped files. Force a fresh row instead. - if (!incomingClearsLastModified(incoming.args)) { + // the swapped files. Force a fresh row instead. The same holds for every + // pass a realm publish enqueues (`awaitedByPublish`), with or without the + // clear: the publish has just swapped in new files, and an already-running + // pass may have read the file mtimes before that swap. + if ( + !incomingClearsLastModified(incoming.args) && + !argsAwaitedByPublish(incoming.args) + ) { for (let candidate of inFlightCandidates) { if (candidate.jobType === incoming.jobType) { return { type: 'join', jobId: candidate.id }; From 04d61137e5e3b4aa0b59f82d57c5d07a929788f0 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 9 Oct 2026 13:45:49 +0200 Subject: [PATCH 2/3] Render every card on republish when a row depends on another realm, and render errored rows again Address review: a published row that depends on a card or module in another realm is not invalidated when that realm changes, so such a realm renders every card on republish. Rows whose HTML holds a render error get their mtime cleared so the pass renders them again. Document that query results are not dependencies, and add tests that pin the same-second touch, the process-start guard, the other-realm check and the render-error invalidation. Co-Authored-By: Claude Opus 5.5 --- .../handlers/handle-publish-realm.ts | 85 +++++++- .../tests/publish-unpublish-realm-test.ts | 196 +++++++++++++++++- 2 files changed, 272 insertions(+), 9 deletions(-) diff --git a/packages/realm-server/handlers/handle-publish-realm.ts b/packages/realm-server/handlers/handle-publish-realm.ts index fe13562fd19..a4f360af905 100644 --- a/packages/realm-server/handlers/handle-publish-realm.ts +++ b/packages/realm-server/handlers/handle-publish-realm.ts @@ -15,6 +15,9 @@ import { userInitiatedPriority, deriveRealmName, notifyAllFileChanges, + baseRealm, + effectiveHasError, + type VirtualNetwork, } from '@cardstack/runtime-common'; import { getUnlistedSlug } from '../lib/unlisted-realm-path.ts'; import { getPublishedRealmDomainOverrides } from '@cardstack/runtime-common/constants'; @@ -327,6 +330,67 @@ async function restoreRealmConfigMtime( await utimes(publishedRealmJsonPath, atime, mtime); } +// The index compares only this realm's own files, so nothing renders a +// published row again when a card or module it depends on in another realm +// changes (the catalog, or a different workspace). Base realm code ships with +// a deploy, which PROCESS_STARTED_AT covers, and so does the code outside any +// realm that rows depend on (host packages, icons, CDN modules). +async function dependsOnOtherRealms( + dbAdapter: DBAdapter, + virtualNetwork: VirtualNetwork, + publishedRealmURL: string, +): Promise { + let deps = ( + (await query(dbAdapter, [ + `SELECT DISTINCT d.dep FROM boxel_index i, jsonb_array_elements_text(i.deps) AS d(dep) + WHERE i.is_deleted IS NOT TRUE AND i.realm_url =`, + param(publishedRealmURL), + `AND NOT starts_with(d.dep,`, + param(publishedRealmURL), + `)`, + ])) as { dep: string }[] + ).map(({ dep }) => dep); + if (deps.length === 0) { + return false; + } + let otherRealms = ( + (await query(dbAdapter, [`SELECT url FROM realm_registry`])) as { + url: string; + }[] + ) + .map(({ url }) => url) + .filter((url) => url !== publishedRealmURL && url !== baseRealm.url); + return deps.some((dep) => { + // A prefix-form dependency (`@cardstack/catalog/…`) names its realm only + // through the virtual network. + let href = dep.startsWith('@') + ? virtualNetwork.realmForReference(dep) + : dep; + return ( + href !== undefined && otherRealms.some((realm) => href.startsWith(realm)) + ); + }); +} + +// The index's mtime check reads only boxel_index.has_error, but a render +// error (a render timeout, a render that throws, a visit failure past the +// reconcile sweep's retry cap) lives on prerendered_html, and the row keeps it +// until its URL is invalidated again. Clear the mtime of every row whose +// effective error state is set, so the pass renders it again even though its +// file did not change. +async function invalidateErroredRows( + dbAdapter: DBAdapter, + publishedRealmURL: string, +): Promise { + await query(dbAdapter, [ + `UPDATE boxel_index AS i SET last_modified = NULL + FROM prerendered_html AS ph + WHERE ph.url = i.url AND ph.realm_url = i.realm_url AND ph.type = i.type + AND ${effectiveHasError()} AND i.realm_url =`, + param(publishedRealmURL), + ]); +} + // The index keeps file mtimes in whole seconds. A file written in the second // the previous publish copied it, and written again later in that same // second, keeps its mtime in the index and reads as unchanged. Stamp each @@ -701,16 +765,28 @@ export default function handlePublishRealm({ // URL is the same as last time, so the rows of unchanged cards // already hold it. Render every card when that is not enough: a // new realm has no rows; realm.json feeds realm info (og:title and - // more) into cards that do not depend on it in the index; and code + // more) into cards that do not depend on it in the index; code // deployed since the last publish can render the same files - // differently. + // differently; and rows that read from another realm can be out of + // date however this realm's files changed. + // + // Query results are not dependencies, so a card that lists other + // cards through a query does not render again when only the + // matching cards change. A full render does not make those results + // current either: its queries run against the committed index, + // which does not yet hold the rows the pass writes. let renderEveryCard = isNewRealm || previousPublishedAt < PROCESS_STARTED_AT || !sameRealmConfig( previousRealmConfig, await readRealmConfig(publishedRealmPath), - ); + ) || + (await dependsOnOtherRealms( + dbAdapter, + virtualNetwork, + publishedRealmURL, + )); // Clear stale modules cache for the published realm (including // error entries from a previous publish) before the reindex's @@ -790,6 +866,9 @@ export default function handlePublishRealm({ // fullIndex below tracks completion for readiness. clearLastModified // forces every row to re-render even where its mtime did not // change. + if (!renderEveryCard) { + await invalidateErroredRows(dbAdapter, publishedRealmURL); + } await enqueueReindexRealmJob( publishedRealmURL, realmUsername, diff --git a/packages/realm-server/tests/publish-unpublish-realm-test.ts b/packages/realm-server/tests/publish-unpublish-realm-test.ts index 48d7c82b459..a7c55abf6a6 100644 --- a/packages/realm-server/tests/publish-unpublish-realm-test.ts +++ b/packages/realm-server/tests/publish-unpublish-realm-test.ts @@ -1527,14 +1527,27 @@ module(basename(import.meta.filename), function () { }, }, }); - // Card files written well before the publish, so their mtimes are - // not in the second the publish starts its copy (see + // The realm server runs in this process, so this is the moment its + // publish handler compares a realm's last publish against. + let processStartedAt = () => Date.now() - process.uptime() * 1000; + // Card files written before the server process started, so their + // mtimes are not in the second a publish starts its copy (see // touchFilesModifiedSince in the publish handler). - let writeCard = (fileName: string, name: string) => { + let writeCard = ( + fileName: string, + name: string, + mtime = new Date(processStartedAt() - 60_000), + ) => { let path = join(sourceRealmFsPath, fileName); writeJsonSync(path, cardJson(name)); - let past = new Date(Date.now() - 60_000); - utimesSync(path, past, past); + utimesSync(path, mtime, mtime); + }; + let lastPublishedAt = async () => { + let [row] = (await dbAdapter.execute( + `SELECT last_published_at FROM realm_registry WHERE url = $1`, + { bind: [publishedRealmURL] }, + )) as { last_published_at: string }[]; + return Number(row.last_published_at); }; hooks.beforeEach(async function () { @@ -1617,7 +1630,11 @@ module(basename(import.meta.filename), function () { await publish(); let unchangedBefore = await readCardRow('unchanged-card'); - writeCard('changed-card.json', 'Changed card, second version'); + writeCard( + 'changed-card.json', + 'Changed card, second version', + new Date(), + ); await publish(); let changedAfter = await readCardRow('changed-card'); @@ -1663,6 +1680,173 @@ module(basename(import.meta.filename), function () { 'the unchanged card is rendered again', ); }); + + test('it renders a card again that changed in the second the previous publish copied it', async function (assert) { + assert.timeout(300_000); + writeCard('changed-card.json', 'Changed card, first version'); + await publish(); + + // The index holds the first version's mtime as the second the + // publish started its copy, and the second version is written in + // that same second, so both mtimes read the same in the index. + let copySecond = Math.floor((await lastPublishedAt()) / 1000); + await dbAdapter.execute( + `UPDATE boxel_index SET last_modified = $1 WHERE realm_url = $2 AND url = $3`, + { + bind: [ + copySecond, + publishedRealmURL, + `${publishedRealmURL}changed-card.json`, + ], + }, + ); + writeCard( + 'changed-card.json', + 'Changed card, second version', + new Date(copySecond * 1000 + 500), + ); + await publish(); + + assert.true( + (await readCardRow('changed-card')).head.includes( + 'Changed card, second version', + ), + 'the changed card is rendered again', + ); + }); + + test('it renders every card again when the last publish was before the server process started', async function (assert) { + assert.timeout(300_000); + writeCard('unchanged-card.json', 'Unchanged card'); + await publish(); + let unchangedBefore = await readCardRow('unchanged-card'); + + await dbAdapter.execute( + `UPDATE realm_registry SET last_published_at = $1 WHERE url = $2`, + { + bind: [Math.floor(processStartedAt()) - 1000, publishedRealmURL], + }, + ); + await publish(); + + assert.notStrictEqual( + (await readCardRow('unchanged-card')).renderedAt, + unchangedBefore.renderedAt, + 'the unchanged card is rendered again', + ); + }); + + test('it renders an unchanged card again when its HTML holds a render error', async function (assert) { + assert.timeout(300_000); + writeCard('unchanged-card.json', 'Unchanged card'); + writeCard('other-card.json', 'Other card'); + await publish(); + let otherBefore = await readCardRow('other-card'); + + // A render error lands on prerendered_html only; the card's + // boxel_index row stays clean. + await dbAdapter.execute( + `UPDATE prerendered_html SET error_doc = $1::jsonb WHERE realm_url = $2 AND url = $3`, + { + bind: [ + JSON.stringify({ message: 'planted render error' }), + publishedRealmURL, + `${publishedRealmURL}unchanged-card.json`, + ], + }, + ); + await publish(); + + let [row] = (await dbAdapter.execute( + `SELECT error_doc FROM prerendered_html WHERE realm_url = $1 AND url = $2`, + { + bind: [ + publishedRealmURL, + `${publishedRealmURL}unchanged-card.json`, + ], + }, + )) as { error_doc: unknown }[]; + assert.strictEqual( + row.error_doc, + null, + 'the render error is replaced by a fresh render', + ); + assert.strictEqual( + (await readCardRow('other-card')).renderedAt, + otherBefore.renderedAt, + 'a card without an error is not rendered again', + ); + }); + + test('it renders every card again when a card depends on another realm', async function (assert) { + assert.timeout(300_000); + let otherRealmResponse = await request + .post('/_create-realm') + .set('Accept', 'application/vnd.api+json') + .set('Content-Type', 'application/json') + .set( + 'Authorization', + `Bearer ${createRealmServerJWT( + { user: ownerUserId, sessionRoom: 'session-room-test' }, + realmSecretSeed, + )}`, + ) + .send( + JSON.stringify({ + data: { + type: 'realm', + attributes: { + name: 'Other Realm', + endpoint: `other-realm-${uuidv4()}`, + }, + }, + }), + ); + let otherRealmURL: string = otherRealmResponse.body.data.id; + await dbAdapter.execute( + `INSERT INTO realm_user_permissions (realm_url, username, read, write, realm_owner) + VALUES ($1, '*', true, false, false)`, + { bind: [otherRealmURL] }, + ); + copySync( + join(fixtureDir('simple'), 'person.gts'), + join( + dir.name, + 'realm_server_3', + ...new URL(otherRealmURL).pathname.split('/').filter(Boolean), + 'person.gts', + ), + ); + writeCard('unchanged-card.json', 'Unchanged card'); + let personPath = join( + sourceRealmFsPath, + 'person-from-other-realm.json', + ); + writeJsonSync(personPath, { + data: { + type: 'card', + attributes: { firstName: 'Other realm person' }, + meta: { + adoptsFrom: { + module: `${otherRealmURL}person`, + name: 'Person', + }, + }, + }, + }); + let past = new Date(processStartedAt() - 60_000); + utimesSync(personPath, past, past); + await publish(); + let unchangedBefore = await readCardRow('unchanged-card'); + + await publish(); + + assert.notStrictEqual( + (await readCardRow('unchanged-card')).renderedAt, + unchangedBefore.renderedAt, + 'the unchanged card is rendered again', + ); + }); }); // The in-process readiness gates only see indexing this instance From 6ed4c7cdaf04c243b1b2740e06ea2fb00b9cf094 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 9 Oct 2026 13:46:47 +0200 Subject: [PATCH 3/3] Document that a skipped card keeps the realm's previous lastPublishedAt Co-Authored-By: Claude Opus 5.5 --- packages/realm-server/handlers/handle-publish-realm.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/realm-server/handlers/handle-publish-realm.ts b/packages/realm-server/handlers/handle-publish-realm.ts index a4f360af905..b2edea426ec 100644 --- a/packages/realm-server/handlers/handle-publish-realm.ts +++ b/packages/realm-server/handlers/handle-publish-realm.ts @@ -774,7 +774,11 @@ export default function handlePublishRealm({ // cards through a query does not render again when only the // matching cards change. A full render does not make those results // current either: its queries run against the committed index, - // which does not yet hold the rows the pass writes. + // which does not yet hold the rows the pass writes. In the same + // way, a card that renders `realmInfo.lastPublishedAt` keeps the + // time of the publish that last rendered it; no base template + // shows it for a published realm, and rendering every card on + // every publish to keep it current is the cost this skip avoids. let renderEveryCard = isNewRealm || previousPublishedAt < PROCESS_STARTED_AT ||