Isolate per-file indexing failures instead of aborting the whole realm

packages/runtime-common/index-runner.ts

@@ -8,6 +8,7 @@ import {
   logger,
   hasCardExtension,
   hasExecutableExtension,
+  isCardResource,
   SupportedMimeType,
   jobIdentity,
   Deferred,
@@ -18,6 +19,7 @@ import {
   type LooseCardResource,
   type InstanceEntry,
   type InstanceErrorIndexEntry,
+  type FileErrorIndexEntry,
   type RealmInfo,
   type FromScratchResult,
   type IncrementalResult,
@@ -33,7 +35,12 @@ import { moduleFrom } from './code-ref.ts';
 import type { RealmResourceIdentifier } from './realm-identifiers.ts';
 import type { CacheScope, DefinitionLookup } from './definition-lookup.ts';
 import type { VirtualNetwork } from './virtual-network.ts';
-import { isCardError } from './error.ts';
+import {
+  CardError,
+  coerceErrorMessage,
+  isCardError,
+  serializableError,
+} from './error.ts';
 import type { IndexingProgressEvent } from './worker.ts';
 import { canonicalURL } from './index-runner/dependency-url.ts';
 import { IndexRunnerDependencyManager } from './index-runner/dependency-resolver.ts';
@@ -544,8 +551,69 @@ export class IndexRunner {
         this.#log.info(
           `${jobIdentity(this.#jobInfo)} tried to visit file ${url.href}, but it no longer exists`,
         );
-      } else {
-        throw err;
+        return;
+      }
+      // A transport-level failure (prerender timeout/abort, network error)
+      // never reaches performCardIndexing/performFileIndexing's own
+      // error-entry construction — visitFileForIndexingFused rethrows
+      // before calling indexCardWithResult/indexFileWithResults. Left
+      // uncaught here, one file's failure propagates out of the
+      // fromScratch/incremental visit loop, skips batch.done(), and
+      // discards every other successfully-visited file's rows for the
+      // whole job. Persist a file-error row instead so the failure is
+      // isolated to this URL, matching the error_doc pattern used for
+      // in-band render errors.
+      let message = coerceErrorMessage(
+        err,
+        `Indexing failed for ${url.href} with no error message (${jobIdentity(this.#jobInfo)})`,
+      );
+      this.#log.warn(
+        `${jobIdentity(this.#jobInfo)} failed to index ${url.href}, recording file-error: ${message}`,
+      );
+      let error = isCardError(err)
+        ? serializableError(err)
+        : serializableError(
+            Object.assign(new CardError(message, { status: 500 }), {
+              stack: (err as Error)?.stack,
+            }),
+          );
+      error.message = message;
+      let fileEntry: FileErrorIndexEntry = {
+        type: 'file-error',
+        error,
+      };
+      await this.batch.updateEntry(url, fileEntry);
+      this.#dependencyResolver.invalidateRelationshipDependencyRowCache(url);
+      this.stats.fileErrors++;
+      // `Batch.invalidate()` already tombstoned every type this URL
+      // previously had in the index — for an existing card that's both
+      // `instance` and `file`. Overwriting only the `file` tombstone
+      // above would let batch.done() promote the untouched `instance`
+      // tombstone, silently removing a previously-good card from search
+      // over a transient error. Re-parse the file ourselves (the throw
+      // above lost visitFileForIndexingFused's internal determination)
+      // and write a matching instance-error row when it's a card, so
+      // the last-known-good instance survives the same as the in-band
+      // render-error path.
+      if (url.href.endsWith('.json')) {
+        try {
+          let fileRef = await this.#reader.readFile(url);
+          let resource = fileRef?.content
+            ? (JSON.parse(fileRef.content)?.data as unknown)
+            : undefined;
+          if (resource && isCardResource(resource)) {
+            let instanceEntry: InstanceErrorIndexEntry = {
+              type: 'instance-error',
+              error,
+            };
+            await this.batch.updateEntry(url, instanceEntry);
+            this.stats.instanceErrors++;
+          }
+        } catch (parseErr) {
+          this.#log.warn(
+            `${jobIdentity(this.#jobInfo)} could not determine whether ${url.href} is a card instance after its visit failed: ${(parseErr as Error)?.message}`,
+          );
+        }
       }
     }
   }