fix(outbox): fail the run after maintenance when a handler module could not load

Author: waleedlatif1

apps/sim/lib/core/outbox/enqueue.test.ts

@@ -27,7 +27,14 @@ const IDLE_MINUTE = new Date('2026-09-16T12:34:45Z')
 const MAINTENANCE_MINUTE = new Date('2026-09-16T12:35:10Z')
 
 const INLINE_OUTPUT = {
-  result: { processed: 0, retried: 0, deadLettered: 0, leaseLost: 0, reaped: 0 },
+  result: {
+    processed: 0,
+    retried: 0,
+    deadLettered: 0,
+    leaseLost: 0,
+    reaped: 0,
+    unloadedEventTypes: [],
+  },
   recoveredDocuments: 0,
   reapedBackgroundWork: 0,
 }
@@ -84,7 +91,14 @@ describe('outbox processor enqueue', () => {
   it('preserves synchronous processing for self-hosted deployments without Trigger', async () => {
     setEnvFlags({ isTriggerDevEnabled: false })
     const output = {
-      result: { processed: 4, retried: 0, deadLettered: 0, leaseLost: 0, reaped: 0 },
+      result: {
+        processed: 4,
+        retried: 0,
+        deadLettered: 0,
+        leaseLost: 0,
+        reaped: 0,
+        unloadedEventTypes: [],
+      },
       recoveredDocuments: 2,
       reapedBackgroundWork: 1,
     }

apps/sim/lib/core/outbox/processor.test.ts

@@ -24,7 +24,14 @@ import { runOutboxProcessor } from '@/lib/core/outbox/processor'
 const mockProcessOutboxEvents = outboxServiceMockFns.mockProcessOutboxEvents
 
 describe('outbox processor recovery', () => {
-  const result = { processed: 5, retried: 1, deadLettered: 0, leaseLost: 0, reaped: 0 }
+  const result = {
+    processed: 5,
+    retried: 1,
+    deadLettered: 0,
+    leaseLost: 0,
+    reaped: 0,
+    unloadedEventTypes: [],
+  }
 
   beforeEach(() => {
     vi.resetAllMocks()
@@ -86,4 +93,18 @@ describe('outbox processor recovery', () => {
     expect(mocks.recover).not.toHaveBeenCalled()
     expect(mocks.reap).not.toHaveBeenCalled()
   })
+
+  it('finishes maintenance, then fails the run naming event types whose handler module failed to load', async () => {
+    mockProcessOutboxEvents.mockResolvedValueOnce({
+      ...result,
+      unloadedEventTypes: ['test.broken', 'test.broken-too'],
+    })
+
+    await expect(runOutboxProcessor()).rejects.toThrow(
+      'Outbox handler modules failed to load; left pending: test.broken, test.broken-too'
+    )
+    expect(mocks.recover).toHaveBeenCalledOnce()
+    expect(mocks.reap).toHaveBeenCalledOnce()
+    expect(mocks.prune).toHaveBeenCalledOnce()
+  })
 })

apps/sim/lib/core/outbox/processor.ts

@@ -71,5 +71,11 @@ export async function runOutboxProcessor(): Promise<OutboxProcessorResult> {
     prunedEvents,
     durationMs: Date.now() - startedAt,
   })
+  /** Fail the run so a broken handler module stays as visible as the crash its static import caused. */
+  if (result.unloadedEventTypes.length > 0) {
+    throw new Error(
+      `Outbox handler modules failed to load; left pending: ${result.unloadedEventTypes.join(', ')}`
+    )
+  }
   return output
 }

apps/sim/lib/core/outbox/service.test.ts

@@ -637,6 +637,7 @@ describe('processOutboxEvents — lazy handler groups', () => {
       processed: 1,
       retried: 0,
       deadLettered: 0,
+      unloadedEventTypes: ['test.flaky'],
     })
     expect(healthyHandler).toHaveBeenCalledOnce()
     expect(claimedEventTypes()).not.toContain('test.flaky')
@@ -646,7 +647,10 @@ describe('processOutboxEvents — lazy handler groups', () => {
     queuePendingEvents([makePendingRow({ eventType: 'test.flaky' })])
     holdLease()
 
-    expect(await processOutboxEvents(groups)).toMatchObject({ processed: 1 })
+    expect(await processOutboxEvents(groups)).toMatchObject({
+      processed: 1,
+      unloadedEventTypes: [],
+    })
     expect(recoveredHandler).toHaveBeenCalledOnce()
   })
 

apps/sim/lib/core/outbox/service.ts

@@ -166,6 +166,8 @@ export interface ProcessOutboxResult {
   deadLettered: number
   leaseLost: number
   reaped: number
+  /** Ready event types left pending because their handler module failed to import. */
+  unloadedEventTypes: string[]
 }
 
 export type ProcessSingleOutboxResult =
@@ -457,7 +459,7 @@ export async function processOutboxEvents(
     reaped = await reapStuckProcessingRows()
     phase = 'discover'
     const readyTypes = await db.execute<{ eventType: string }>(readyEventTypesQuery(new Date()))
-    const { handlers, eligibleTypes } = await resolveOutboxHandlers(
+    const { handlers, eligibleTypes, unloadedEventTypes } = await resolveOutboxHandlers(
       handlerGroups,
       readyTypes.map(({ eventType }) => eventType)
     )
@@ -497,7 +499,7 @@ export async function processOutboxEvents(
       else retried++
     }
 
-    return { processed, retried, deadLettered, leaseLost, reaped }
+    return { processed, retried, deadLettered, leaseLost, reaped, unloadedEventTypes }
   } catch (error) {
     logger.error('Outbox processing failed', {
       phase,
@@ -517,13 +519,17 @@ export async function processOutboxEvents(
  * Imports the groups that serve any ready event type and returns the types to claim this run. A
  * group whose import fails leaves its event types unclaimed: unlike a missing handler, which
  * spends an attempt and eventually dead-letters, a failed import says nothing about the events,
- * so they stay pending for a later run. Event types outside every group stay eligible and reach
- * the missing-handler path.
+ * so they stay pending for a later run and are reported as `unloadedEventTypes`. Event types
+ * outside every group stay eligible and reach the missing-handler path.
  */
 async function resolveOutboxHandlers(
   groups: readonly LazyOutboxHandlerGroup[],
   readyEventTypes: string[]
-): Promise<{ handlers: OutboxHandlerRegistry; eligibleTypes: string[] }> {
+): Promise<{
+  handlers: OutboxHandlerRegistry
+  eligibleTypes: string[]
+  unloadedEventTypes: string[]
+}> {
   const unavailableEventTypes = new Set<string>()
   const ready = new Set(readyEventTypes)
   const dueGroups = groups.filter((group) => group.events.some((eventType) => ready.has(eventType)))
@@ -547,6 +553,7 @@ async function resolveOutboxHandlers(
   return {
     handlers,
     eligibleTypes: readyEventTypes.filter((eventType) => !unavailableEventTypes.has(eventType)),
+    unloadedEventTypes: readyEventTypes.filter((eventType) => unavailableEventTypes.has(eventType)),
   }
 }