Skip to content

Render only changed cards when republishing a realm #6624

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jurgenwerk merged 3 commits into from
Oct 12, 2026
Merged

Render only changed cards when republishing a realm #6624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cbd45c9
Render only changed cards when republishing a realm
jurgenwerk Oct 9, 2026
04d6113
Render every card on republish when a row depends on another realm, a…
jurgenwerk Oct 9, 2026
6ed4c7c
Document that a skipped card keeps the realm's previous lastPublishedAt
jurgenwerk Oct 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 210 additions & 12 deletions packages/realm-server/handlers/handle-publish-realm.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ import {
userInitiatedPriority,
deriveRealmName,
notifyAllFileChanges,
baseRealm,
effectiveHasError,
type VirtualNetwork,
} from '@cardstack/runtime-common';
import { getUnlistedSlug } from '../lib/unlisted-realm-path.ts';
import { getPublishedRealmDomainOverrides } from '@cardstack/runtime-common/constants';
Expand All @@ -24,7 +27,18 @@ import fsExtra from 'fs-extra';
// Async fs ops only: the publish handler runs these inside the request, and a
// synchronous copy/move of a whole realm directory would freeze the Node event
// loop, stalling every other concurrent request until it finished.
const { copy, readJson, writeJson, remove, pathExists, move } = fsExtra;
const {
copy,
readJson,
writeJson,
remove,
pathExists,
move,
readFile,
readdir,
stat,
utimes,
} = fsExtra;

import {
fetchRequestFromContext,
Expand Down Expand Up @@ -278,6 +292,127 @@ async function dropRealmPolicyPointer(realmPath: string): Promise<void> {
await writeJson(realmJsonPath, realmConfigDoc, { spaces: 2 });
}

// A deploy restarts this process. A realm last published before this moment
// was rendered by host and base code that may have changed since, so its
// republish renders every card again. A restart without a deploy costs one
// such full render too.
const PROCESS_STARTED_AT = Date.now() - process.uptime() * 1000;

async function readRealmConfig(realmPath: string): Promise<Buffer | undefined> {
let realmJsonPath = join(realmPath, 'realm.json');
return (await pathExists(realmJsonPath))
? await readFile(realmJsonPath)
: undefined;
}

function sameRealmConfig(a: Buffer | undefined, b: Buffer | undefined) {
return a === undefined || b === undefined ? a === b : a.equals(b);
}

// The copy keeps every file's mtime, so the index skips the files that did
// not change since the last publish. The publish then rewrites the copied
// realm.json (dropRealmPolicyPointer, ensureRealmIndexBoilerplateOptIn), and
// each write stamps it with the current time. Give it the source file's mtime
// back, so an unchanged realm.json also reads as unchanged.
async function restoreRealmConfigMtime(
sourceRealmPath: string,
publishedRealmPath: string,
): Promise<void> {
let sourceRealmJsonPath = join(sourceRealmPath, 'realm.json');
let publishedRealmJsonPath = join(publishedRealmPath, 'realm.json');
if (
!(await pathExists(sourceRealmJsonPath)) ||
!(await pathExists(publishedRealmJsonPath))
) {
return;
}
let { atime, mtime } = await stat(sourceRealmJsonPath);
await utimes(publishedRealmJsonPath, atime, mtime);
}

// The index compares only this realm's own files, so nothing renders a
// published row again when a card or module it depends on in another realm
// changes (the catalog, or a different workspace). Base realm code ships with
// a deploy, which PROCESS_STARTED_AT covers, and so does the code outside any
// realm that rows depend on (host packages, icons, CDN modules).
async function dependsOnOtherRealms(
dbAdapter: DBAdapter,
virtualNetwork: VirtualNetwork,
publishedRealmURL: string,
): Promise<boolean> {
let deps = (
(await query(dbAdapter, [
`SELECT DISTINCT d.dep FROM boxel_index i, jsonb_array_elements_text(i.deps) AS d(dep)
WHERE i.is_deleted IS NOT TRUE AND i.realm_url =`,
param(publishedRealmURL),
`AND NOT starts_with(d.dep,`,
param(publishedRealmURL),
`)`,
])) as { dep: string }[]
).map(({ dep }) => dep);
if (deps.length === 0) {
return false;
}
let otherRealms = (
(await query(dbAdapter, [`SELECT url FROM realm_registry`])) as {
url: string;
}[]
)
.map(({ url }) => url)
.filter((url) => url !== publishedRealmURL && url !== baseRealm.url);
return deps.some((dep) => {
// A prefix-form dependency (`@cardstack/catalog/…`) names its realm only
// through the virtual network.
let href = dep.startsWith('@')
? virtualNetwork.realmForReference(dep)
: dep;
return (
href !== undefined && otherRealms.some((realm) => href.startsWith(realm))
);
});
}

// The index's mtime check reads only boxel_index.has_error, but a render
// error (a render timeout, a render that throws, a visit failure past the
// reconcile sweep's retry cap) lives on prerendered_html, and the row keeps it
// until its URL is invalidated again. Clear the mtime of every row whose
// effective error state is set, so the pass renders it again even though its
// file did not change.
async function invalidateErroredRows(
dbAdapter: DBAdapter,
publishedRealmURL: string,
): Promise<void> {
await query(dbAdapter, [
`UPDATE boxel_index AS i SET last_modified = NULL
FROM prerendered_html AS ph
WHERE ph.url = i.url AND ph.realm_url = i.realm_url AND ph.type = i.type
AND ${effectiveHasError()} AND i.realm_url =`,
param(publishedRealmURL),
]);
}

// The index keeps file mtimes in whole seconds. A file written in the second
// the previous publish copied it, and written again later in that same
// second, keeps its mtime in the index and reads as unchanged. Stamp each
// file modified at or after the second the previous publish started its copy
// with the current time, so the index renders it again. Such a file renders
// once more at the next republish, since its index row then holds the stamp
// and not the source mtime.
async function touchFilesModifiedSince(

@jurgenwerk jurgenwerk Oct 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Claude Code 🤖] No test pins this function or the PROCESS_STARTED_AT guard. I disabled the touchFilesModifiedSince call, and all 24 publish tests still passed. The existing republish tests hit the same-second case only when a publish and its index finish in less than one second.

Suggested tests in the republishing a realm module:

  • Set both versions of a card file to an mtime inside the second of the previous last_published_at (with utimesSync), republish, and assert that the card renders again.
  • Set realm_registry.last_published_at to 0 before a republish, and assert that an unchanged card renders again.

Test coverage, non-blocking.

@jurgenwerk jurgenwerk Oct 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Claude Code 🤖] Fixed in 04d61137e5. Two new tests: it renders a card again that changed in the second the previous publish copied it (plants the same-second mtime in boxel_index, then writes the second version in that second) and it renders every card again when the last publish was before the server process started. With touchFilesModifiedSince disabled, only the first test fails. With the process-start guard disabled, only the second test fails.

dir: string,
since: number,
now = new Date(),
): Promise<void> {
for (let entry of await readdir(dir, { withFileTypes: true })) {
let path = join(dir, entry.name);
if (entry.isDirectory()) {
await touchFilesModifiedSince(path, since, now);
} else if (entry.isFile() && (await stat(path)).mtimeMs >= since) {
await utimes(path, now, now);
}
}
}

export default function handlePublishRealm({
dbAdapter,
definitionLookup,
Expand Down Expand Up @@ -484,12 +619,16 @@ export default function handlePublishRealm({
// mounts the (re-)published realm on its first request. The
// response is 202 Accepted with status:'pending'; the client polls
// /<publishedRealmURL>/_readiness-check to learn when it's ready.
let { lastPublishedAt, publishedRealmId, isNewRealm } =
let { lastPublishedAt, publishedRealmId, isNewRealm, renderEveryCard } =
await dbAdapter.withWriteLock(publishedRealmURL, async () => {
let existingRows = (await query(dbAdapter, [
`SELECT disk_id, owner_username FROM realm_registry WHERE kind = 'published' AND url =`,
`SELECT disk_id, owner_username, last_published_at FROM realm_registry WHERE kind = 'published' AND url =`,
param(publishedRealmURL),
])) as { disk_id: string; owner_username: string }[];
])) as {
disk_id: string;
owner_username: string;
last_published_at: string | number | null;
}[];
let isNewRealm = existingRows.length === 0;

let publishedRealmId: string;
Expand Down Expand Up @@ -533,6 +672,12 @@ export default function handlePublishRealm({
let sourceRealmPath = sourceRealm.dir;
let publishedDir = join(realmsRootPath, PUBLISHED_DIRECTORY_NAME);
let publishedRealmPath = join(publishedDir, publishedRealmId);
let previousRealmConfig = isNewRealm
? undefined
: await readRealmConfig(publishedRealmPath);
let previousPublishedAt = Number(
existingRows[0]?.last_published_at ?? 0,
);

// Copy source to a temporary directory first, then swap it into
// place so that a failed copy doesn't destroy the existing
Expand All @@ -547,7 +692,19 @@ export default function handlePublishRealm({
let backupPath = `${publishedRealmPath}.backup`;
await remove(tempCopyPath);
await remove(backupPath);
await copy(sourceRealmPath, tempCopyPath);
// Recorded before the copy, so the next republish knows from which
// second on a copied file's mtime is ambiguous (see
// touchFilesModifiedSince).
let publishStartedAt = Date.now();
await copy(sourceRealmPath, tempCopyPath, {
preserveTimestamps: true,
});
if (!isNewRealm) {
await touchFilesModifiedSince(
tempCopyPath,
Math.floor(previousPublishedAt / 1000) * 1000,
);
}
try {
await dropRealmPolicyPointer(tempCopyPath);
if (await pathExists(publishedRealmPath)) {
Expand Down Expand Up @@ -601,6 +758,39 @@ export default function handlePublishRealm({
// card (/realm.json) on disk before the reindex below picks
// it up.
await ensureRealmIndexBoilerplateOptIn(publishedRealmPath);
await restoreRealmConfigMtime(sourceRealmPath, publishedRealmPath);

// A republish renders again only the files that changed since the
// last publish, and the files that depend on them. The published
// URL is the same as last time, so the rows of unchanged cards
// already hold it. Render every card when that is not enough: a
// new realm has no rows; realm.json feeds realm info (og:title and
// more) into cards that do not depend on it in the index; code
// deployed since the last publish can render the same files
// differently; and rows that read from another realm can be out of
// date however this realm's files changed.
//
// Query results are not dependencies, so a card that lists other
// cards through a query does not render again when only the
// matching cards change. A full render does not make those results
// current either: its queries run against the committed index,
// which does not yet hold the rows the pass writes. In the same
// way, a card that renders `realmInfo.lastPublishedAt` keeps the
// time of the publish that last rendered it; no base template
// shows it for a published realm, and rendering every card on
// every publish to keep it current is the cost this skip avoids.
let renderEveryCard =

@richardhjtan richardhjtan Oct 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Claude Code 🤖] Blocking: a card whose published HTML is an error keeps that error across republishes. With clearLastModified false, discoverInvalidations re-renders a row only when indexEntry.hasError is set or its mtime moved, and getModifiedTimes reads boxel_index.has_error alone. An error recorded on the HTML channel lives in prerendered_html.error_doc (a render timeout, a deterministic render error, a visit failure past the reconcile sweep's retry cap), while that card's boxel_index row stays clean. The reconcile sweep treats such rows as terminal until the URL's next invalidation, and a full republish was that invalidation. With this change the card stays broken on the published site until its file changes or the server restarts.

Could the republish also render rows whose HTML channel holds an error? For example, invalidate the URLs with prerendered_html.error_doc IS NOT NULL at the realm's current generation (the effectiveHasError condition) alongside the mtime diff, or set renderEveryCard when any exist. A test that plants an HTML error row for an unchanged card and asserts that a republish renders it again would pin it.

@jurgenwerk jurgenwerk Oct 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Claude Code 🤖] Fixed in 04d61137e5. When a republish does not render every card, invalidateErroredRows clears last_modified before the job is enqueued, for each row of the published realm where effectiveHasError() holds. It reuses the same condition the query engine reads, so the two cannot drift. The pass then renders those rows again. The new test it renders an unchanged card again when its HTML holds a render error plants a prerendered_html.error_doc on an unchanged card. It asserts that a republish replaces the error, and that a card without an error is not rendered again. It fails without the invalidation.

isNewRealm ||
previousPublishedAt < PROCESS_STARTED_AT ||

@chatgpt-codex-connector chatgpt-codex-connector Bot Oct 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Re-render cards when foreign dependencies change

When an unchanged card imports or links to content in another public realm and that dependency changes without restarting this process, the copied files and realm.json retain the same mtimes and bytes, so this predicate leaves renderEveryCard false. Cross-realm changes explicitly do not propagate to the consuming realm's index (packages/runtime-common/realm.ts:13033-13043), meaning the from-scratch pass skips the card and preserves stale prerendered HTML until a local edit or restart. Track upstream dependency generations or force affected cards to render on republish.

Useful? React with 👍 / 👎.

@jurgenwerk jurgenwerk Oct 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Claude Code 🤖] Fixed in 04d61137e5: dependsOnOtherRealms makes a republish render every card when a published row depends on another realm (see the reply on the renderEveryCard thread above).

Comment on lines +782 to +784

@chatgpt-codex-connector chatgpt-codex-connector Bot Oct 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Account for the new publish timestamp when deciding to render

Every republish updates realm_registry.last_published_at before indexing, and that value is part of the RealmInfo embedded in card responses (packages/runtime-common/realm.ts:16478-16482). A card component that renders realmInfo.lastPublishedAt therefore has new output even when its files and realm.json are unchanged, but renderEveryCard remains false and its existing prerendered HTML is retained; cards otherwise invalidated in the same publish can consequently show a different publication time from skipped cards. This runtime metadata change needs to invalidate consumers or be included in the full-render decision.

Useful? React with 👍 / 👎.

@jurgenwerk jurgenwerk Oct 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Claude Code 🤖] Not changed in code; documented in 6ed4c7cdaf. To keep realmInfo.lastPublishedAt current, every card would have to render on every publish, and that is the cost this change removes. No base template shows the value for a published realm: workspace.gts reads only the source-realm map shape. The comment above renderEveryCard now states that a skipped card keeps the time of the publish that last rendered it.

!sameRealmConfig(
previousRealmConfig,
await readRealmConfig(publishedRealmPath),
) ||
(await dependsOnOtherRealms(
dbAdapter,
virtualNetwork,
publishedRealmURL,
));

// Clear stale modules cache for the published realm (including
// error entries from a previous publish) before the reindex's
Expand All @@ -617,7 +807,7 @@ export default function handlePublishRealm({
// their own next invalidation arrived.
await definitionLookup.clearRealmDefinitions(publishedRealmURL);

let lastPublishedAt = Date.now().toString();
let lastPublishedAt = publishStartedAt.toString();
try {
await upsertPublishedRealmInRegistry(dbAdapter, {
publishedRealmURL,
Expand Down Expand Up @@ -678,8 +868,11 @@ export default function handlePublishRealm({
// lazy-mounts it and start()'s from-scratch pass coalesces with
// this job. For a republish already mounted here, the post-lock
// fullIndex below tracks completion for readiness. clearLastModified
// forces every row to re-render even though file copies preserve
// mtimes.
// forces every row to re-render even where its mtime did not
// change.
if (!renderEveryCard) {
await invalidateErroredRows(dbAdapter, publishedRealmURL);
}
await enqueueReindexRealmJob(
publishedRealmURL,
realmUsername,
Expand All @@ -689,10 +882,15 @@ export default function handlePublishRealm({
// This publish is blocked on the published realm's HTML (readiness
// gates on it), so the prerender-html job this pass spawns runs
// co-equal with indexing rather than one tier below.
{ clearLastModified: true, awaitedByPublish: true },
{ clearLastModified: renderEveryCard, awaitedByPublish: true },
);

return { lastPublishedAt, publishedRealmId, isNewRealm };
return {
lastPublishedAt,
publishedRealmId,
isNewRealm,
renderEveryCard,
};
});

// Mount the published realm on this instance so it is served as soon as
Expand Down Expand Up @@ -723,7 +921,7 @@ export default function handlePublishRealm({
// Republish: the realm already has index rows, so start() does NOT
// re-index them and #startedUp resolves without reflecting the swapped
// files — readinessCheck must instead wait on indexing(). Register a
// tracked clearLastModified reindex SYNCHRONOUSLY (before the 202) so
// tracked reindex SYNCHRONOUSLY (before the 202) so
// indexing() reflects it and readiness can't report ready before the
// reindex lands. Get the mounted realm, or mount it first when this
// instance is cold (e.g. after a restart, or the publish landed on an
Expand All @@ -738,7 +936,7 @@ export default function handlePublishRealm({
if (publishedRealm) {
void publishedRealm
.fullIndex(userInitiatedPriority, {
clearLastModified: true,
clearLastModified: renderEveryCard,
// Republish is awaiting this HTML for readiness, so its render
// runs co-equal with indexing (see prerenderHtmlPriority).
awaitedByPublish: true,
Expand Down
Loading
Loading