From 6ae5858e0734f69c48646baceda6e4e833ebe5da Mon Sep 17 00:00:00 2001 From: Richard Tan Date: Thu, 2 Jul 2026 22:08:18 +0800 Subject: [PATCH 1/2] Isolate per-file indexing failures instead of aborting the whole realm tryToVisit rethrew any non-404 error from a file visit, which unwound the entire fromScratch/incremental visit loop and skipped batch.done(). Since batch.done() is what promotes the working table into the live boxel_index, a single file's transport-level failure (e.g. a prerender timeout) discarded every other already-successfully-indexed file in the same job, leaving the realm mounted with an empty or stale search index and no error surfaced. Catch non-404 errors per file and record a file-error entry for that URL instead, mirroring the error_doc pattern already used for in-band render errors, so the rest of the realm still indexes and commits. --- packages/runtime-common/index-runner.ts | 43 +++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 334c0239d58..239b4a857be 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -18,6 +18,7 @@ import { type LooseCardResource, type InstanceEntry, type InstanceErrorIndexEntry, + type FileErrorIndexEntry, type RealmInfo, type FromScratchResult, type IncrementalResult, @@ -33,7 +34,12 @@ import { moduleFrom } from './code-ref.ts'; import type { RealmResourceIdentifier } from './realm-identifiers.ts'; import type { CacheScope, DefinitionLookup } from './definition-lookup.ts'; import type { VirtualNetwork } from './virtual-network.ts'; -import { isCardError } from './error.ts'; +import { + CardError, + coerceErrorMessage, + isCardError, + serializableError, +} from './error.ts'; import type { IndexingProgressEvent } from './worker.ts'; import { canonicalURL } from './index-runner/dependency-url.ts'; import { IndexRunnerDependencyManager } from './index-runner/dependency-resolver.ts'; @@ -544,9 +550,40 @@ export class IndexRunner { this.#log.info( `${jobIdentity(this.#jobInfo)} tried to visit file ${url.href}, but it no longer exists`, ); - } else { - throw err; + return; } + // A transport-level failure (prerender timeout/abort, network error) + // never reaches performCardIndexing/performFileIndexing's own + // error-entry construction — visitFileForIndexingFused rethrows + // before calling indexCardWithResult/indexFileWithResults. Left + // uncaught here, one file's failure propagates out of the + // fromScratch/incremental visit loop, skips batch.done(), and + // discards every other successfully-visited file's rows for the + // whole job. Persist a file-error row instead so the failure is + // isolated to this URL, matching the error_doc pattern used for + // in-band render errors. + let message = coerceErrorMessage( + err, + `Indexing failed for ${url.href} with no error message (${jobIdentity(this.#jobInfo)})`, + ); + this.#log.warn( + `${jobIdentity(this.#jobInfo)} failed to index ${url.href}, recording file-error: ${message}`, + ); + let error = isCardError(err) + ? serializableError(err) + : serializableError( + Object.assign(new CardError(message, { status: 500 }), { + stack: (err as Error)?.stack, + }), + ); + error.message = message; + let entry: FileErrorIndexEntry = { + type: 'file-error', + error, + }; + await this.batch.updateEntry(url, entry); + this.#dependencyResolver.invalidateRelationshipDependencyRowCache(url); + this.stats.fileErrors++; } } From 5e88b86aba6117c139fe6660bc3e2af465bad3e8 Mon Sep 17 00:00:00 2001 From: Richard Tan Date: Thu, 2 Jul 2026 22:31:53 +0800 Subject: [PATCH 2/2] Preserve last-known-good card instances on visit failures Batch.invalidate() tombstones every existing type at a URL up front, including the instance row for a previously-indexed card. The prior commit's file-error entry only overwrote the file tombstone, so batch.done() would promote the untouched instance tombstone and silently remove an existing card from search over a transient error (e.g. a prerender timeout) instead of preserving it with an error row the way the in-band render-error path does. Re-check whether the failed URL is a card and, if so, also write an instance-error entry alongside the file-error one. --- packages/runtime-common/index-runner.ts | 35 +++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 239b4a857be..d607644986c 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -8,6 +8,7 @@ import { logger, hasCardExtension, hasExecutableExtension, + isCardResource, SupportedMimeType, jobIdentity, Deferred, @@ -577,13 +578,43 @@ export class IndexRunner { }), ); error.message = message; - let entry: FileErrorIndexEntry = { + let fileEntry: FileErrorIndexEntry = { type: 'file-error', error, }; - await this.batch.updateEntry(url, entry); + await this.batch.updateEntry(url, fileEntry); this.#dependencyResolver.invalidateRelationshipDependencyRowCache(url); this.stats.fileErrors++; + // `Batch.invalidate()` already tombstoned every type this URL + // previously had in the index — for an existing card that's both + // `instance` and `file`. Overwriting only the `file` tombstone + // above would let batch.done() promote the untouched `instance` + // tombstone, silently removing a previously-good card from search + // over a transient error. Re-parse the file ourselves (the throw + // above lost visitFileForIndexingFused's internal determination) + // and write a matching instance-error row when it's a card, so + // the last-known-good instance survives the same as the in-band + // render-error path. + if (url.href.endsWith('.json')) { + try { + let fileRef = await this.#reader.readFile(url); + let resource = fileRef?.content + ? (JSON.parse(fileRef.content)?.data as unknown) + : undefined; + if (resource && isCardResource(resource)) { + let instanceEntry: InstanceErrorIndexEntry = { + type: 'instance-error', + error, + }; + await this.batch.updateEntry(url, instanceEntry); + this.stats.instanceErrors++; + } + } catch (parseErr) { + this.#log.warn( + `${jobIdentity(this.#jobInfo)} could not determine whether ${url.href} is a card instance after its visit failed: ${(parseErr as Error)?.message}`, + ); + } + } } }