feat(ensindexer): leverage SWR caches to achieve goals related to LocalPonderClient

apps/ensindexer/ponder/src/api/handlers/ensnode-api.ts

@@ -19,12 +19,11 @@ import { buildOmnichainIndexingStatusSnapshot } from "@/lib/indexing-status-buil
 import { getLocalPonderClient } from "@/lib/ponder-api-client";
 
 const app = new Hono();
-
-// Calling `getLocalPonderClient` at the top level to initialize
-// the singleton client instance on app startup.
-// This ensures that the client is ready to use when handling requests,
-// and allows us to catch initialization errors early.
-getLocalPonderClient();
+// Get the local Ponder Client instance
+// Note that the client initialization is designed to be non-blocking,
+// by implementing internal SWR caches with proactive revalidation to
+// load necessary data for the client state.
+const localPonderClient = getLocalPonderClient();
 
 // include ENSIndexer Public Config endpoint
 app.get("/config", async (c) => {
@@ -42,7 +41,6 @@ app.get("/indexing-status", async (c) => {
   let omnichainSnapshot: OmnichainIndexingStatusSnapshot | undefined;
 
   try {
-    const localPonderClient = await getLocalPonderClient();
     const chainsIndexingMetadata = await localPonderClient.chainsIndexingMetadata();
 
     omnichainSnapshot = buildOmnichainIndexingStatusSnapshot(chainsIndexingMetadata);

apps/ensindexer/ponder/src/api/lib/cache/chains-indexing-metadata-immutable.cache.ts

@@ -0,0 +1,86 @@
+import type { PublicClient } from "viem";
+
+import { SWRCache } from "@ensnode/ensnode-sdk";
+import type { BlockrangeWithStartBlock, ChainId, PonderIndexingMetrics } from "@ensnode/ponder-sdk";
+
+import {
+  buildChainsIndexingMetadataImmutable,
+  type ChainsIndexingMetadataImmutable,
+} from "../chains-indexing-metadata-immutable";
+import type { PonderClientCache } from "./ponder-client.cache";
+
+/**
+ * Context required to load the chains indexing metadata immutable cache.
+ */
+export interface ChainsIndexingMetadataImmutableCacheContext {
+  indexedChainIds: Set<ChainId>;
+  chainsConfigBlockrange: Map<ChainId, BlockrangeWithStartBlock>;
+  publicClients: Map<ChainId, PublicClient>;
+  ponderClientCache: PonderClientCache;
+}
+
+export type ChainsIndexingMetadataImmutableCache = SWRCache<
+  ChainsIndexingMetadataImmutable,
+  ChainsIndexingMetadataImmutableCacheContext
+>;
+
+/**
+ * Cache for the immutable metadata of the indexed chains.
+ *
+ * This cache is designed to store metadata that is expected to remain constant
+ * throughout the indexing process. The metadata is build based on
+ * {@link PonderIndexingMetrics} value cached in {@link PonderClientCache}.
+ * There may be a few failed attempts to load this cache at the startup of
+ * the Ponder app until the metrics become available. Once the data is
+ * successfully loaded, the cache stops proactive revalidation since the data
+ * is expected to be immutable.
+ */
+export const chainsIndexingMetadataImmutableCache = new SWRCache({
+  fn: async function loadChainsIndexingMetadataImmutable(_cachedValue, context) {
+    if (!context) {
+      throw new Error(
+        `ChainsIndexingMetadataImmutableCache context must be set to load Chains Indexing Metadata Immutable`,
+      );
+    }
+
+    const { indexedChainIds, chainsConfigBlockrange, publicClients, ponderClientCache } = context;
+
+    try {
+      console.info(`[ChainsIndexingMetadataImmutableCache]: loading data...`);
+      const ponderClientCacheResult = await ponderClientCache.read();
+
+      // Invariant: indexing metrics must be available in cache
+      if (ponderClientCacheResult instanceof Error) {
+        throw new Error(
+          `Ponder Indexing Metrics must be available in cache to build chains indexing metadata immutable: ${ponderClientCacheResult.message}`,
+        );
+      }
+
+      const { ponderIndexingMetrics } = ponderClientCacheResult;
+
+      const chainsIndexingMetadataImmutable = await buildChainsIndexingMetadataImmutable(
+        indexedChainIds,
+        chainsConfigBlockrange,
+        publicClients,
+        ponderIndexingMetrics,
+      );
+
+      console.info(`[ChainsIndexingMetadataImmutableCache]: Successfully loaded data`);
+
+      // Stop the proactive revalidation of this cache since we have
+      // successfully loaded the data and initialized the client state.
+      chainsIndexingMetadataImmutableCache.stopProactiveRevalidation();
+
+      return chainsIndexingMetadataImmutable;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.error(
+        `[ChainsIndexingMetadataImmutableCache]: an error occurred while loading data: ${errorMessage}`,
+      );
+
+      throw new Error(`Failed to load Chains Indexing Metadata Immutable: ${errorMessage}`);
+    }
+  },
+  ttl: Number.POSITIVE_INFINITY,
+  proactiveRevalidationInterval: 5,
+}) satisfies ChainsIndexingMetadataImmutableCache;

apps/ensindexer/ponder/src/api/lib/cache/ponder-client.cache.ts

@@ -0,0 +1,62 @@
+import { SWRCache } from "@ensnode/ensnode-sdk";
+import type {
+  PonderClient,
+  PonderIndexingMetrics,
+  PonderIndexingStatus,
+} from "@ensnode/ponder-sdk";
+
+/**
+ * Context required to load the Ponder Client cache.
+ */
+export interface PonderClientCacheContext {
+  ponderClient: PonderClient;
+}
+
+export interface PonderClientCacheResult {
+  ponderIndexingMetrics: PonderIndexingMetrics;
+  ponderIndexingStatus: PonderIndexingStatus;
+}
+
+export type PonderClientCache = SWRCache<PonderClientCacheResult, PonderClientCacheContext>;
+
+/**
+ * Cache for Ponder Client, including its indexing metrics and status for each
+ * indexed chain.
+ *
+ * Ponder Indexing Metrics and Ponder Indexing Status can both change frequently,
+ * so the cache is designed to proactively revalidate data to ensure freshness.
+ *
+ * Note, that Ponder app needs a while at startup to populate indexing metrics,
+ * and indexing status, so a few of the initial attempts to load this cache may
+ * fail until required data is made available by the Ponder app.
+ */
+export const ponderClientCache = new SWRCache({
+  fn: async function loadPonderClientCache(_cachedValue, context) {
+    if (!context) {
+      throw new Error(
+        `PonderClientCache context must be set to load Ponder Indexing Metrics and Status`,
+      );
+    }
+
+    const { ponderClient } = context;
+
+    try {
+      console.info(`[PonderClientCache]: loading data...`);
+      const [ponderIndexingMetrics, ponderIndexingStatus] = await Promise.all([
+        ponderClient.metrics(),
+        ponderClient.status(),
+      ]);
+      console.info(`[PonderClientCache]: Successfully loaded data`);
+
+      return { ponderIndexingMetrics, ponderIndexingStatus };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.error(`[PonderClientCache]: an error occurred while loading data: ${errorMessage}`);
+
+      throw new Error(`Failed to load Ponder Client cache: ${errorMessage}`);
+    }
+  },
+  ttl: Number.POSITIVE_INFINITY,
+  proactiveRevalidationInterval: 5, // indexing metrics and status can change frequently, so proactively revalidate every 5 seconds to ensure data is fresh
+  proactivelyInitialize: true,
+}) satisfies PonderClientCache;

apps/ensindexer/ponder/src/api/lib/chains-indexing-metadata-dynamic.ts

@@ -0,0 +1,49 @@
+import type { ChainId, PonderIndexingMetrics, PonderIndexingStatus } from "@ensnode/ponder-sdk";
+
+import type { ChainIndexingMetadataDynamic } from "@/lib/indexing-status-builder/chain-indexing-metadata";
+
+/**
+ * Build a map of chain ID to its dynamic indexing metadata.
+ *
+ * The dynamic metadata is based on the current indexing metrics and status
+ * of the chain, which can change over time as the chain is being indexed.
+ *
+ * @param indexedChainIds A set of chain IDs that are being indexed.
+ * @param ponderIndexingMetrics The current indexing metrics for all chains.
+ * @param ponderIndexingStatus The current indexing status for all chains.
+ *
+ * @returns A map of chain ID to its dynamic indexing metadata.
+ *
+ * @throws Error if any invariants are violated.
+ */
+export function buildChainsIndexingMetadataDynamic(
+  indexedChainIds: Set<ChainId>,
+  ponderIndexingMetrics: PonderIndexingMetrics,
+  ponderIndexingStatus: PonderIndexingStatus,
+): Map<ChainId, ChainIndexingMetadataDynamic> {
+  const chainsIndexingMetadataDynamic = new Map<ChainId, ChainIndexingMetadataDynamic>();
+
+  for (const chainId of indexedChainIds.values()) {
+    const chainIndexingMetrics = ponderIndexingMetrics.chains.get(chainId);
+    const chainIndexingStatus = ponderIndexingStatus.chains.get(chainId);
+
+    // Invariants: indexing metrics and indexing status must exist in proper state for the indexed chain.
+    if (!chainIndexingMetrics) {
+      throw new Error(`Indexing metrics must be available for indexed chain ID ${chainId}`);
+    }
+
+    if (!chainIndexingStatus) {
+      throw new Error(`Indexing status must be available for indexed chain ID ${chainId}`);
+    }
+
+    const metadataDynamic = {
+      indexingMetrics: chainIndexingMetrics,
+      indexingStatus: chainIndexingStatus,
+    } satisfies ChainIndexingMetadataDynamic;
+
+    // Cache the dynamic metadata for this chain ID
+    chainsIndexingMetadataDynamic.set(chainId, metadataDynamic);
+  }
+
+  return chainsIndexingMetadataDynamic;
+}

apps/ensindexer/ponder/src/api/lib/chains-indexing-metadata-immutable.ts

@@ -0,0 +1,112 @@
+import type { PublicClient } from "viem";
+
+import { createIndexingConfig } from "@ensnode/ensnode-sdk";
+import {
+  type BlockRef,
+  type BlockrangeWithStartBlock,
+  type ChainId,
+  ChainIndexingStates,
+  type PonderIndexingMetrics,
+} from "@ensnode/ponder-sdk";
+
+import type { ChainIndexingMetadataImmutable } from "@/lib/indexing-status-builder/chain-indexing-metadata";
+
+import { fetchBlockRef } from "./fetch-block-ref";
+
+/**
+ * Build an immutable indexing metadata for a chain.
+ *
+ * Some of the metadata fields are based on RPC calls to fetch block references.
+ *
+ * @param startBlock Chain's start block.
+ * @param endBlock Chain's end block (optional).
+ * @param backfillEndBlock Chain's backfill end block.
+ *
+ * @returns The immutable indexing metadata for the chain.
+ */
+function buildChainIndexingMetadataImmutable(
+  startBlock: BlockRef,
+  endBlock: BlockRef | null,
+  backfillEndBlock: BlockRef,
+): ChainIndexingMetadataImmutable {
+  const chainIndexingConfig = createIndexingConfig(startBlock, endBlock);
+
+  return {
+    backfillScope: {
+      startBlock,
+      endBlock: backfillEndBlock,
+    },
+    indexingConfig: chainIndexingConfig,
+  };
+}
+
+/**
+ * Map of chain ID to its immutable indexing metadata.
+ */
+export type ChainsIndexingMetadataImmutable = Map<ChainId, ChainIndexingMetadataImmutable>;
+
+/**
+ * Build a map of chain ID to its immutable indexing metadata.
+ *
+ * @param publicClients A map of chain ID to its corresponding public client,
+ *                      used to fetch block references for chain's blockrange.
+ * @param ponderClient The Ponder client used to fetch indexing metrics and status.
+ * @returns A map of chain ID to its immutable indexing metadata.
+ * @throws Error if any of the required data cannot be fetched or is invalid,
+ *         or if invariants are violated.
+ */
+export async function buildChainsIndexingMetadataImmutable(
+  indexedChainIds: Set<ChainId>,
+  chainsConfigBlockrange: Map<ChainId, BlockrangeWithStartBlock>,
+  publicClients: Map<ChainId, PublicClient>,
+  ponderIndexingMetrics: PonderIndexingMetrics,
+): Promise<Map<ChainId, ChainIndexingMetadataImmutable>> {
+  const chainsIndexingMetadataImmutable = new Map<ChainId, ChainIndexingMetadataImmutable>();
+
+  for (const chainId of indexedChainIds.values()) {
+    const chainConfigBlockrange = chainsConfigBlockrange.get(chainId);
+    const chainIndexingMetrics = ponderIndexingMetrics.chains.get(chainId);
+    const publicClient = publicClients.get(chainId);
+
+    // Invariants: chain config blockrange, indexing metrics, and public client
+    // must exist in proper state for the indexed chain.
+    if (!chainConfigBlockrange) {
+      throw new Error(`Chain config blockrange must be available for indexed chain ID ${chainId}`);
+    }
+
+    if (!chainIndexingMetrics) {
+      throw new Error(`Indexing metrics must be available for indexed chain ID ${chainId}`);
+    }
+
+    if (chainIndexingMetrics.state !== ChainIndexingStates.Historical) {
+      throw new Error(
+        `Chain indexing state must be "historical" for indexed chain ID ${chainId}, but got "${chainIndexingMetrics.state}"`,
+      );
+    }
+
+    if (!publicClient) {
+      throw new Error(`Public client must be available for indexed chain ID ${chainId}`);
+    }
+
+    const backfillEndBlockNumber =
+      chainConfigBlockrange.startBlock + chainIndexingMetrics.historicalTotalBlocks - 1;
+
+    const [startBlock, endBlock, backfillEndBlock] = await Promise.all([
+      fetchBlockRef(publicClient, chainConfigBlockrange.startBlock),
+      chainConfigBlockrange.endBlock
+        ? fetchBlockRef(publicClient, chainConfigBlockrange.endBlock)
+        : null,
+      fetchBlockRef(publicClient, backfillEndBlockNumber),
+    ]);
+
+    const metadataImmutable = buildChainIndexingMetadataImmutable(
+      startBlock,
+      endBlock,
+      backfillEndBlock,
+    );
+
+    chainsIndexingMetadataImmutable.set(chainId, metadataImmutable);
+  }
+
+  return chainsIndexingMetadataImmutable;
+}