From 2cac72bf844436a786bd1a44b1a4e52d912ded1d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 23:48:56 -0700 Subject: [PATCH 1/5] improvement(outbox): load handler modules only for the event types a run will process --- .../api/cron/reconcile-billing-seats/route.ts | 2 +- .../api/v1/admin/subscriptions/[id]/route.ts | 2 +- .../lib/notification-events.ts | 1 + .../ee/access-requests/lib/notifications.ts | 8 +- .../application/admit-sync.ts | 11 +- .../application/content-outbox-event.ts | 1 + .../application/content-outbox.ts | 5 +- .../ee/workspace-forking/lib/create-fork.ts | 3 +- .../lib/admin/invitation-operation-event.ts | 1 + .../lib/admin/invitation-operation.test.ts | 2 +- apps/sim/lib/admin/invitation-operation.ts | 2 +- apps/sim/lib/admin/member-operation-event.ts | 1 + apps/sim/lib/admin/member-operation.test.ts | 1 - apps/sim/lib/admin/member-operation.ts | 8 +- apps/sim/lib/admin/subscription-lifecycle.ts | 2 +- apps/sim/lib/billing/cycle-close.ts | 2 +- .../billing/enterprise-owner-claim-events.ts | 2 + .../sim/lib/billing/enterprise-owner-claim.ts | 6 +- .../lib/billing/enterprise-provisioning.ts | 14 +- .../membership-reconciliation-event.ts | 1 + .../membership-reconciliation.ts | 6 +- .../lib/billing/organizations/membership.ts | 5 +- .../pause-pro-for-coverage.test.ts | 2 +- .../billing/organizations/provision-seat.ts | 2 +- apps/sim/lib/billing/organizations/seats.ts | 2 +- apps/sim/lib/billing/threshold-billing.ts | 2 +- .../lib/billing/validation/seat-management.ts | 2 +- .../sim/lib/billing/webhooks/outbox-events.ts | 23 ++ .../billing/webhooks/outbox-handlers.test.ts | 3 +- .../lib/billing/webhooks/outbox-handlers.ts | 25 +- apps/sim/lib/core/outbox/handlers.test.ts | 65 +++++ apps/sim/lib/core/outbox/handlers.ts | 240 ++++++++++++++++++ apps/sim/lib/core/outbox/processor.test.ts | 36 +-- apps/sim/lib/core/outbox/processor.ts | 42 +-- apps/sim/lib/core/outbox/service.test.ts | 99 ++++++++ apps/sim/lib/core/outbox/service.ts | 66 ++++- ...bedding-processing-recovery.integration.ts | 2 +- ...rovider-processing-recovery.integration.ts | 2 +- .../purged-detach-reservation.integration.ts | 6 +- .../search-source-progress.integration.ts | 2 +- .../storage-accounting.integration.ts | 6 +- .../application/slack-search/outbox.ts | 2 +- .../application/slack-search/turn-event.ts | 1 + .../application/slack-search/turns.ts | 2 +- .../lib/knowledge/connectors/deletion.test.ts | 6 +- apps/sim/lib/knowledge/connectors/deletion.ts | 2 +- .../knowledge/connectors/detachment.test.ts | 2 +- .../lib/knowledge/connectors/detachment.ts | 2 +- .../lib/knowledge/connectors/outbox-events.ts | 2 + .../knowledge/documents/checkpoint-events.ts | 2 + .../document-processing-source.test.ts | 2 +- .../documents/embedding-checkpoints.test.ts | 2 +- .../documents/embedding-checkpoints.ts | 2 +- .../documents/ocr-checkpoints.test.ts | 2 +- .../knowledge/documents/ocr-checkpoints.ts | 3 +- .../processing-continuation-dispatch.ts | 3 +- .../processing-continuation-event.ts | 1 + .../documents/processing-outbox-handler.ts | 28 +- .../sim/lib/mothership/inbox/cleanup-event.ts | 1 + .../lib/mothership/inbox/cleanup-outbox.ts | 2 +- .../organizations/resource-cleanup-event.ts | 1 + .../organizations/resource-cleanup.test.ts | 2 +- .../sim/lib/organizations/resource-cleanup.ts | 2 +- .../file-versions.integration.ts | 2 +- .../contexts/workspace/file-outbox-events.ts | 2 + .../workspace-file-live-doc-outbox.test.ts | 6 +- .../workspace-file-live-doc-outbox.ts | 3 +- ...kspace-file-storage-cleanup-outbox.test.ts | 2 +- .../workspace-file-storage-cleanup-outbox.ts | 3 +- .../workflows/application/mapped-import.ts | 3 +- .../lib/workflows/deployment-outbox-events.ts | 8 + .../lib/workflows/deployment-outbox.test.ts | 2 +- apps/sim/lib/workflows/deployment-outbox.ts | 10 +- apps/sim/lib/workspaces/admin-move-event.ts | 1 + apps/sim/lib/workspaces/admin-move.test.ts | 2 +- apps/sim/lib/workspaces/admin-move.ts | 2 +- .../workspaces/operations/outbox-events.ts | 3 + apps/sim/lib/workspaces/operations/outbox.ts | 11 +- .../src/mocks/billing-outbox-handlers.mock.ts | 7 +- .../src/mocks/organization-membership.mock.ts | 6 +- scripts/check-unused-exports.baseline.json | 3 - 81 files changed, 631 insertions(+), 230 deletions(-) create mode 100644 apps/sim/ee/workspace-forking/application/content-outbox-event.ts create mode 100644 apps/sim/lib/admin/invitation-operation-event.ts create mode 100644 apps/sim/lib/admin/member-operation-event.ts create mode 100644 apps/sim/lib/billing/enterprise-owner-claim-events.ts create mode 100644 apps/sim/lib/billing/organizations/membership-reconciliation-event.ts create mode 100644 apps/sim/lib/billing/webhooks/outbox-events.ts create mode 100644 apps/sim/lib/core/outbox/handlers.test.ts create mode 100644 apps/sim/lib/core/outbox/handlers.ts create mode 100644 apps/sim/lib/knowledge/application/slack-search/turn-event.ts create mode 100644 apps/sim/lib/knowledge/connectors/outbox-events.ts create mode 100644 apps/sim/lib/knowledge/documents/checkpoint-events.ts create mode 100644 apps/sim/lib/knowledge/documents/processing-continuation-event.ts create mode 100644 apps/sim/lib/mothership/inbox/cleanup-event.ts create mode 100644 apps/sim/lib/organizations/resource-cleanup-event.ts create mode 100644 apps/sim/lib/uploads/contexts/workspace/file-outbox-events.ts create mode 100644 apps/sim/lib/workflows/deployment-outbox-events.ts create mode 100644 apps/sim/lib/workspaces/admin-move-event.ts create mode 100644 apps/sim/lib/workspaces/operations/outbox-events.ts diff --git a/apps/sim/app/api/cron/reconcile-billing-seats/route.ts b/apps/sim/app/api/cron/reconcile-billing-seats/route.ts index 133ae7378d4..64e1e957ee6 100644 --- a/apps/sim/app/api/cron/reconcile-billing-seats/route.ts +++ b/apps/sim/app/api/cron/reconcile-billing-seats/route.ts @@ -3,7 +3,7 @@ import { toError } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { verifyCronAuth } from '@/lib/auth/internal' import { reconcileTeamSeatDrift } from '@/lib/billing/organizations/seat-drift' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { findDeadLetteredEvents } from '@/lib/core/outbox/service' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' diff --git a/apps/sim/app/api/v1/admin/subscriptions/[id]/route.ts b/apps/sim/app/api/v1/admin/subscriptions/[id]/route.ts index f462ec0b5af..ffc61afe9ba 100644 --- a/apps/sim/app/api/v1/admin/subscriptions/[id]/route.ts +++ b/apps/sim/app/api/v1/admin/subscriptions/[id]/route.ts @@ -33,7 +33,7 @@ import { } from '@/lib/api/contracts/v1/admin' import { parseRequest } from '@/lib/api/server' import { requireStripeClient } from '@/lib/billing/stripe-client' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { withAdminAuthParams } from '@/app/api/v1/admin/middleware' diff --git a/apps/sim/ee/access-requests/lib/notification-events.ts b/apps/sim/ee/access-requests/lib/notification-events.ts index 3adf1aa1897..05ac9cf391f 100644 --- a/apps/sim/ee/access-requests/lib/notification-events.ts +++ b/apps/sim/ee/access-requests/lib/notification-events.ts @@ -1,2 +1,3 @@ export const PERMISSION_ACCESS_REQUEST_CREATED_EVENT = 'permission-access-request.created' export const PERMISSION_ACCESS_REQUEST_DECIDED_EVENT = 'permission-access-request.decided' +export const PERMISSION_ACCESS_REQUEST_NOTIFY_ADMIN_EVENT = 'permission-access-request.notify-admin' diff --git a/apps/sim/ee/access-requests/lib/notifications.ts b/apps/sim/ee/access-requests/lib/notifications.ts index 28d2bf48012..ab5e76cb5e1 100644 --- a/apps/sim/ee/access-requests/lib/notifications.ts +++ b/apps/sim/ee/access-requests/lib/notifications.ts @@ -18,12 +18,12 @@ import { getMyAccessRequestHref } from '@/ee/access-requests/lib/navigation' import { PERMISSION_ACCESS_REQUEST_CREATED_EVENT, PERMISSION_ACCESS_REQUEST_DECIDED_EVENT, + PERMISSION_ACCESS_REQUEST_NOTIFY_ADMIN_EVENT, } from '@/ee/access-requests/lib/notification-events' import { isAccessRequestEnabled } from '@/ee/access-requests/lib/settings' const logger = createLogger('PermissionAccessRequestNotifications') const ADMIN_RECIPIENT_PAGE_SIZE = 50 -const ADMIN_NOTIFICATION_EVENT = 'permission-access-request.notify-admin' const notificationPayloadSchema = z.object({ requestId: z.string().min(1).max(256) }).strict() const createdPayloadSchema = notificationPayloadSchema.extend({ afterMemberId: z.string().min(1).max(256).optional(), @@ -176,8 +176,8 @@ export const permissionAccessRequestOutboxHandlers = { .insert(outboxEvent) .values( recipients.map((recipient) => ({ - id: `${ADMIN_NOTIFICATION_EVENT}:${requestId}:${recipient.userId}`, - eventType: ADMIN_NOTIFICATION_EVENT, + id: `${PERMISSION_ACCESS_REQUEST_NOTIFY_ADMIN_EVENT}:${requestId}:${recipient.userId}`, + eventType: PERMISSION_ACCESS_REQUEST_NOTIFY_ADMIN_EVENT, payload: { requestId, recipientUserId: recipient.userId }, })) ) @@ -187,7 +187,7 @@ export const permissionAccessRequestOutboxHandlers = { return continueOutboxHandler('Continue access request administrator notifications') } }, - [ADMIN_NOTIFICATION_EVENT]: async (rawPayload, context) => { + [PERMISSION_ACCESS_REQUEST_NOTIFY_ADMIN_EVENT]: async (rawPayload, context) => { const { requestId, recipientUserId } = adminPayloadSchema.parse(rawPayload) const request = await loadRequest(requestId) if ( diff --git a/apps/sim/ee/workspace-forking/application/admit-sync.ts b/apps/sim/ee/workspace-forking/application/admit-sync.ts index 6d590df1378..f1d291ab0ea 100644 --- a/apps/sim/ee/workspace-forking/application/admit-sync.ts +++ b/apps/sim/ee/workspace-forking/application/admit-sync.ts @@ -2,6 +2,11 @@ import { generateId } from '@sim/utils/id' import { truncate } from '@sim/utils/string' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' import type { DbOrTx } from '@/lib/db/types' +import { + WORKSPACE_MCP_CHANGED_EVENT, + WORKSPACE_OPERATION_OBSERVE_EVENT, + WORKSPACE_WORKFLOWS_CHANGED_EVENT, +} from '@/lib/workspaces/operations/outbox-events' import { insertWorkspaceOperationReceipt, type WorkspaceOperationReport, @@ -100,7 +105,7 @@ export async function admitForkSync( } if (params.mcpAttachmentServerIds.length) report.effectEventIds!.push( - await enqueueOutboxEvent(tx, 'workspace.mcp.changed', { + await enqueueOutboxEvent(tx, WORKSPACE_MCP_CHANGED_EVENT, { serverIds: params.mcpAttachmentServerIds, }) ) @@ -117,10 +122,10 @@ export async function admitForkSync( ? 'completed_with_warnings' : 'completed' } - await enqueueOutboxEvent(tx, 'workspace.workflows.changed', { + await enqueueOutboxEvent(tx, WORKSPACE_WORKFLOWS_CHANGED_EVENT, { workspaceId: params.targetWorkspaceId, }) - await enqueueOutboxEvent(tx, 'workspace.operation.observe', { + await enqueueOutboxEvent(tx, WORKSPACE_OPERATION_OBSERVE_EVENT, { workspaceId: report.workspaceId, operationId: report.operationId, }) diff --git a/apps/sim/ee/workspace-forking/application/content-outbox-event.ts b/apps/sim/ee/workspace-forking/application/content-outbox-event.ts new file mode 100644 index 00000000000..590be9a72d7 --- /dev/null +++ b/apps/sim/ee/workspace-forking/application/content-outbox-event.ts @@ -0,0 +1 @@ +export const FORK_CONTENT_COPY_EVENT = 'workspace.fork.content.copy' diff --git a/apps/sim/ee/workspace-forking/application/content-outbox.ts b/apps/sim/ee/workspace-forking/application/content-outbox.ts index be7e28df692..b3b35207d03 100644 --- a/apps/sim/ee/workspace-forking/application/content-outbox.ts +++ b/apps/sim/ee/workspace-forking/application/content-outbox.ts @@ -11,6 +11,7 @@ import { } from '@/lib/core/outbox/service' import type { DbOrTx } from '@/lib/db/types' import type { WorkspaceOperationReport } from '@/lib/workspaces/operations/receipts' +import { FORK_CONTENT_COPY_EVENT } from '@/ee/workspace-forking/application/content-outbox-event' import { type ForkContentCopyPayload, runForkContentCopy, @@ -153,11 +154,11 @@ export async function enqueueDurableForkContent( }) if (Buffer.byteLength(JSON.stringify(payload)) > 8 * 1024 * 1024) throw new OrchestrationError('payload_too_large', 'Fork background work exceeds 8 MiB') - return enqueueOutboxEvent(tx, 'workspace.fork.content.copy', payload) + return enqueueOutboxEvent(tx, FORK_CONTENT_COPY_EVENT, payload) } export const forkContentOutboxHandlers = { - 'workspace.fork.content.copy': withOutboxHandlerTimeout(async (raw, context) => { + [FORK_CONTENT_COPY_EVENT]: withOutboxHandlerTimeout(async (raw, context) => { const payload = contentPayloadSchema.parse(raw) const [receipt] = await db .select({ report: workspaceOperationReceipt.report }) diff --git a/apps/sim/ee/workspace-forking/lib/create-fork.ts b/apps/sim/ee/workspace-forking/lib/create-fork.ts index 69c161ed921..341fc16117f 100644 --- a/apps/sim/ee/workspace-forking/lib/create-fork.ts +++ b/apps/sim/ee/workspace-forking/lib/create-fork.ts @@ -16,6 +16,7 @@ import { collectReferencedFileFolderPaths, } from '@/lib/workflows/references/reference-scan' import type { ForkRemapKind } from '@/lib/workflows/references/remap-references' +import { WORKSPACE_OPERATION_OBSERVE_EVENT } from '@/lib/workspaces/operations/outbox-events' import { findWorkspaceOperationReceipt, insertWorkspaceOperationReceipt, @@ -625,7 +626,7 @@ export async function createFork(params: CreateForkParams): Promise organizationMembershipMo vi.mock('@/lib/core/outbox/service', () => outboxServiceMock) import { - ADMIN_INVITATION_OPERATION_EVENT_TYPE, adminInvitationOperationOutboxHandlers, createAdminInvitationOperation, } from '@/lib/admin/invitation-operation' +import { ADMIN_INVITATION_OPERATION_EVENT_TYPE } from '@/lib/admin/invitation-operation-event' const mocks = { enqueue: outboxServiceMockFns.mockEnqueueOutboxEvent, diff --git a/apps/sim/lib/admin/invitation-operation.ts b/apps/sim/lib/admin/invitation-operation.ts index 3b5b3c6fbba..61037f04368 100644 --- a/apps/sim/lib/admin/invitation-operation.ts +++ b/apps/sim/lib/admin/invitation-operation.ts @@ -4,6 +4,7 @@ import { member, outboxEvent, user, workspace } from '@sim/db/schema' import { normalizeEmail } from '@sim/utils/string' import { and, count, eq, inArray, sql } from 'drizzle-orm' import { z } from 'zod' +import { ADMIN_INVITATION_OPERATION_EVENT_TYPE } from '@/lib/admin/invitation-operation-event' import { ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE, enterpriseInvitePeoplePayloadSchema, @@ -21,7 +22,6 @@ import { } from '@/lib/invitations/direct-grant-event' import { MAX_INVITE_EMAILS, MAX_INVITE_WORKSPACES } from '@/lib/invitations/limits' -export const ADMIN_INVITATION_OPERATION_EVENT_TYPE = 'admin.organization-invitation-operation' const MAX_INVITATION_OPERATION_FAILURE_DETAILS = 100 const adminInvitationOperationRequestSchema = z.object({ diff --git a/apps/sim/lib/admin/member-operation-event.ts b/apps/sim/lib/admin/member-operation-event.ts new file mode 100644 index 00000000000..d0924abbde2 --- /dev/null +++ b/apps/sim/lib/admin/member-operation-event.ts @@ -0,0 +1 @@ +export const ADMIN_MEMBER_OPERATION_EVENT_TYPE = 'admin.organization-member-operation' diff --git a/apps/sim/lib/admin/member-operation.test.ts b/apps/sim/lib/admin/member-operation.test.ts index fe0fe8a0d24..efe43489095 100644 --- a/apps/sim/lib/admin/member-operation.test.ts +++ b/apps/sim/lib/admin/member-operation.test.ts @@ -26,7 +26,6 @@ vi.mock('@/lib/billing/organizations/member-limits', () => organizationMemberLim vi.mock('@/lib/billing/organizations/seats', () => organizationSeatsMock) vi.mock('@/lib/billing/core/usage', () => billingUsageMock) vi.mock('@/lib/workspaces/admin-move', () => ({ - MIGRATED_INVITATION_EMAIL_EVENT_TYPE: 'invitation.send-migrated-link', moveWorkspaceToOrganization: hoisted.moveWorkspace, })) vi.mock('@/lib/workspaces/organization-workspaces', () => ({ diff --git a/apps/sim/lib/admin/member-operation.ts b/apps/sim/lib/admin/member-operation.ts index e68090a08e4..3d5d431f4c6 100644 --- a/apps/sim/lib/admin/member-operation.ts +++ b/apps/sim/lib/admin/member-operation.ts @@ -5,6 +5,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { and, count, desc, eq, inArray, sql } from 'drizzle-orm' import { z } from 'zod' +import { ADMIN_MEMBER_OPERATION_EVENT_TYPE } from '@/lib/admin/member-operation-event' import { syncUsageLimitsFromSubscription } from '@/lib/billing/core/usage' import { acquireUserBillingIdentityLock } from '@/lib/billing/organizations/billing-identity-lock' import { setOrgMemberUsageLimit } from '@/lib/billing/organizations/member-limits' @@ -23,13 +24,10 @@ import { outboxPayloadHasSourceOperationId, } from '@/lib/core/outbox/service' import type { DbOrTx } from '@/lib/db/types' -import { - MIGRATED_INVITATION_EMAIL_EVENT_TYPE, - moveWorkspaceToOrganization, -} from '@/lib/workspaces/admin-move' +import { moveWorkspaceToOrganization } from '@/lib/workspaces/admin-move' +import { MIGRATED_INVITATION_EMAIL_EVENT_TYPE } from '@/lib/workspaces/admin-move-event' import { ownedAttachableWorkspacesWhere } from '@/lib/workspaces/organization-workspaces' -export const ADMIN_MEMBER_OPERATION_EVENT_TYPE = 'admin.organization-member-operation' const MEMBER_OPERATION_WORKSPACE_BATCH_SIZE = 10 const ADMIN_API_AUDIT_EMAIL = 'admin-api@internal.simstudio.ai' const logger = createLogger('AdminMemberOperation') diff --git a/apps/sim/lib/admin/subscription-lifecycle.ts b/apps/sim/lib/admin/subscription-lifecycle.ts index 4b9f0c96b15..72335997f92 100644 --- a/apps/sim/lib/admin/subscription-lifecycle.ts +++ b/apps/sim/lib/admin/subscription-lifecycle.ts @@ -7,7 +7,7 @@ import type { AdminMutationActor } from '@/lib/admin/dashboard' import { acquireOrganizationMutationLock } from '@/lib/billing/organizations/membership' import { requireStripeClient } from '@/lib/billing/stripe-client' import { ENTITLED_SUBSCRIPTION_STATUSES } from '@/lib/billing/subscriptions/utils' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' const RECENT_INVOICE_LIMIT = 12 diff --git a/apps/sim/lib/billing/cycle-close.ts b/apps/sim/lib/billing/cycle-close.ts index 952b3cd30a9..1ec3047b314 100644 --- a/apps/sim/lib/billing/cycle-close.ts +++ b/apps/sim/lib/billing/cycle-close.ts @@ -21,7 +21,7 @@ import { computeWeeklyRefreshConsumed } from '@/lib/billing/credits/weekly-refre import { getPlanWeeklyRefreshDollars, isEnterprise, isFree } from '@/lib/billing/plan-helpers' import { ENTITLED_SUBSCRIPTION_STATUSES, getPlanPricing } from '@/lib/billing/subscriptions/utils' import { toDecimal, toNumber } from '@/lib/billing/utils/decimal' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import type { DbOrTx } from '@/lib/db/types' diff --git a/apps/sim/lib/billing/enterprise-owner-claim-events.ts b/apps/sim/lib/billing/enterprise-owner-claim-events.ts new file mode 100644 index 00000000000..ab3de0a407e --- /dev/null +++ b/apps/sim/lib/billing/enterprise-owner-claim-events.ts @@ -0,0 +1,2 @@ +export const ENTERPRISE_OWNER_CLAIM_EVENT_TYPE = 'enterprise.invite-owner' +export const ENTERPRISE_OWNER_ACTIVATION_EVENT_TYPE = 'enterprise.activate-owner-claim' diff --git a/apps/sim/lib/billing/enterprise-owner-claim.ts b/apps/sim/lib/billing/enterprise-owner-claim.ts index 3a9cb50ef00..539547dca4b 100644 --- a/apps/sim/lib/billing/enterprise-owner-claim.ts +++ b/apps/sim/lib/billing/enterprise-owner-claim.ts @@ -10,6 +10,10 @@ import { z } from 'zod' import { getEmailSubject, renderEnterpriseOwnerInvitationEmail } from '@/components/emails' import { parseBillingConcurrencyLimit } from '@/lib/billing/concurrency-defaults' import { dollarsToCredits } from '@/lib/billing/credits/conversion' +import { + ENTERPRISE_OWNER_ACTIVATION_EVENT_TYPE, + ENTERPRISE_OWNER_CLAIM_EVENT_TYPE, +} from '@/lib/billing/enterprise-owner-claim-events' import { assertEnterpriseInvitationEligibility, EnterpriseProvisioningError, @@ -42,8 +46,6 @@ import { } from '@/lib/workspaces/create' import { ownedAttachableWorkspacesWhere } from '@/lib/workspaces/organization-workspaces' -export const ENTERPRISE_OWNER_CLAIM_EVENT_TYPE = 'enterprise.invite-owner' -export const ENTERPRISE_OWNER_ACTIVATION_EVENT_TYPE = 'enterprise.activate-owner-claim' export const ENTERPRISE_OWNER_CLAIM_EXPIRY_DAYS = INVITATION_EXPIRY_DAYS const logger = createLogger('EnterpriseOwnerClaim') diff --git a/apps/sim/lib/billing/enterprise-provisioning.ts b/apps/sim/lib/billing/enterprise-provisioning.ts index bbdd9479cb1..06a82a801f9 100644 --- a/apps/sim/lib/billing/enterprise-provisioning.ts +++ b/apps/sim/lib/billing/enterprise-provisioning.ts @@ -30,10 +30,8 @@ import { sql, } from 'drizzle-orm' import type Stripe from 'stripe' -import { - ADMIN_INVITATION_OPERATION_EVENT_TYPE, - parseAdminInvitationOperationPayload, -} from '@/lib/admin/invitation-operation' +import { parseAdminInvitationOperationPayload } from '@/lib/admin/invitation-operation' +import { ADMIN_INVITATION_OPERATION_EVENT_TYPE } from '@/lib/admin/invitation-operation-event' import { parseBillingConcurrencyLimit } from '@/lib/billing/concurrency-defaults' import { getBillingConcurrencyLimit } from '@/lib/billing/concurrency-limits' import { resolveEnterpriseReportingPeriod } from '@/lib/billing/core/reporting-period' @@ -76,7 +74,7 @@ import { requireStripeClient } from '@/lib/billing/stripe-client' import { TERMINAL_SUBSCRIPTION_STATUSES } from '@/lib/billing/subscriptions/utils' import { countPendingSeatInvitations } from '@/lib/billing/validation/seat-management' import { withEnterpriseReconciliationLease } from '@/lib/billing/webhooks/enterprise-reconciliation-lease' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { env } from '@/lib/core/config/env' import { continueOutboxHandler, @@ -94,10 +92,8 @@ import { createWorkspaceInvitation, prepareWorkspaceInvitationContext, } from '@/lib/invitations/workspace-invitations' -import { - MIGRATED_INVITATION_EMAIL_EVENT_TYPE, - moveWorkspaceToOrganization, -} from '@/lib/workspaces/admin-move' +import { moveWorkspaceToOrganization } from '@/lib/workspaces/admin-move' +import { MIGRATED_INVITATION_EMAIL_EVENT_TYPE } from '@/lib/workspaces/admin-move-event' import { ownedAttachableWorkspacesWhere } from '@/lib/workspaces/organization-workspaces' const TERMINAL_STATUSES = new Set(TERMINAL_SUBSCRIPTION_STATUSES) diff --git a/apps/sim/lib/billing/organizations/membership-reconciliation-event.ts b/apps/sim/lib/billing/organizations/membership-reconciliation-event.ts new file mode 100644 index 00000000000..9fe155e782c --- /dev/null +++ b/apps/sim/lib/billing/organizations/membership-reconciliation-event.ts @@ -0,0 +1 @@ +export const MEMBER_BILLING_RECONCILIATION_EVENT_TYPE = 'billing.reconcile-member-after-org-leave' diff --git a/apps/sim/lib/billing/organizations/membership-reconciliation.ts b/apps/sim/lib/billing/organizations/membership-reconciliation.ts index 2a97a0de0a0..7ac2b7d1d02 100644 --- a/apps/sim/lib/billing/organizations/membership-reconciliation.ts +++ b/apps/sim/lib/billing/organizations/membership-reconciliation.ts @@ -1,8 +1,6 @@ import { syncUsageLimitsFromSubscription } from '@/lib/billing/core/usage' -import { - MEMBER_BILLING_RECONCILIATION_EVENT_TYPE, - restoreUserProSubscription, -} from '@/lib/billing/organizations/membership' +import { restoreUserProSubscription } from '@/lib/billing/organizations/membership' +import { MEMBER_BILLING_RECONCILIATION_EVENT_TYPE } from '@/lib/billing/organizations/membership-reconciliation-event' import type { OutboxHandler } from '@/lib/core/outbox/service' interface MemberBillingReconciliationPayload { diff --git a/apps/sim/lib/billing/organizations/membership.ts b/apps/sim/lib/billing/organizations/membership.ts index e7b25750441..e93f4bcd0e2 100644 --- a/apps/sim/lib/billing/organizations/membership.ts +++ b/apps/sim/lib/billing/organizations/membership.ts @@ -38,6 +38,7 @@ import { } from '@/lib/billing/enterprise-outbox' import { acquireUserBillingIdentityLock } from '@/lib/billing/organizations/billing-identity-lock' import { setOrgMemberUsageLimit } from '@/lib/billing/organizations/member-limits' +import { MEMBER_BILLING_RECONCILIATION_EVENT_TYPE } from '@/lib/billing/organizations/membership-reconciliation-event' import { isPaid, sqlIsPro } from '@/lib/billing/plan-helpers' import { changeOrganizationWorkspaceBilledAccountsInTx } from '@/lib/billing/storage/payer-transfer' import { @@ -46,7 +47,7 @@ import { } from '@/lib/billing/subscriptions/utils' import { toDecimal, toNumber } from '@/lib/billing/utils/decimal' import { validateSeatAvailability } from '@/lib/billing/validation/seat-management' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { isBillingEnabled } from '@/lib/core/config/env-flags' import { OrchestrationError } from '@/lib/core/orchestration/types' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' @@ -75,8 +76,6 @@ const logger = createLogger('OrganizationMembership') const ORG_MEMBERSHIP_LOCK_TIMEOUT_MS = 5_000 -export const MEMBER_BILLING_RECONCILIATION_EVENT_TYPE = 'billing.reconcile-member-after-org-leave' - /** Serializes organization-wide owner, seat, move, and membership decisions. */ export async function acquireOrganizationMutationLock( tx: DbTransaction, diff --git a/apps/sim/lib/billing/organizations/pause-pro-for-coverage.test.ts b/apps/sim/lib/billing/organizations/pause-pro-for-coverage.test.ts index 8354ebd0d07..c29d82c340c 100644 --- a/apps/sim/lib/billing/organizations/pause-pro-for-coverage.test.ts +++ b/apps/sim/lib/billing/organizations/pause-pro-for-coverage.test.ts @@ -10,7 +10,7 @@ vi.mock('@/lib/billing/storage/payer-transfer', () => ({ vi.mock('@/lib/core/outbox/service', () => outboxServiceMock) import { pauseProSubscriptionForOrgCoverage } from '@/lib/billing/organizations/membership' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' const mockEnqueueOutboxEvent = outboxServiceMockFns.mockEnqueueOutboxEvent diff --git a/apps/sim/lib/billing/organizations/provision-seat.ts b/apps/sim/lib/billing/organizations/provision-seat.ts index 5749b34f808..0d5e55d79c5 100644 --- a/apps/sim/lib/billing/organizations/provision-seat.ts +++ b/apps/sim/lib/billing/organizations/provision-seat.ts @@ -16,7 +16,7 @@ import { } from '@/lib/billing/plan-helpers' import { getPlanByName } from '@/lib/billing/plans' import { hasUsableSubscriptionStatus } from '@/lib/billing/subscriptions/utils' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' import type { DbOrTx, DbTransaction } from '@/lib/db/types' diff --git a/apps/sim/lib/billing/organizations/seats.ts b/apps/sim/lib/billing/organizations/seats.ts index f38c3ff3b1d..7b2d02e70d6 100644 --- a/apps/sim/lib/billing/organizations/seats.ts +++ b/apps/sim/lib/billing/organizations/seats.ts @@ -6,7 +6,7 @@ import { and, count, desc, eq, inArray } from 'drizzle-orm' import { syncSubscriptionUsageLimits } from '@/lib/billing/organization' import { isTeam } from '@/lib/billing/plan-helpers' import { ENTITLED_SUBSCRIPTION_STATUSES } from '@/lib/billing/subscriptions/utils' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { isBillingEnabled } from '@/lib/core/config/env-flags' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' import { captureServerEvent } from '@/lib/posthog/server' diff --git a/apps/sim/lib/billing/threshold-billing.ts b/apps/sim/lib/billing/threshold-billing.ts index 3758e692805..1f50d5b2050 100644 --- a/apps/sim/lib/billing/threshold-billing.ts +++ b/apps/sim/lib/billing/threshold-billing.ts @@ -21,7 +21,7 @@ import { isOrgScopedSubscription, } from '@/lib/billing/subscriptions/utils' import { toDecimal, toNumber } from '@/lib/billing/utils/decimal' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { env, envNumber } from '@/lib/core/config/env' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' import { captureServerEvent } from '@/lib/posthog/server' diff --git a/apps/sim/lib/billing/validation/seat-management.ts b/apps/sim/lib/billing/validation/seat-management.ts index c819d2b7656..7c1a42abccc 100644 --- a/apps/sim/lib/billing/validation/seat-management.ts +++ b/apps/sim/lib/billing/validation/seat-management.ts @@ -6,7 +6,7 @@ import { getOrganizationSubscription } from '@/lib/billing/core/billing' import { resolveEnterpriseMetadataIntent } from '@/lib/billing/enterprise-outbox' import { isEnterprise, isFree } from '@/lib/billing/plan-helpers' import { getEffectiveSeats } from '@/lib/billing/subscriptions/utils' -import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import { isBillingEnabled } from '@/lib/core/config/env-flags' import { hasInflightOutboxEvent } from '@/lib/core/outbox/service' import type { DbOrTx } from '@/lib/db/types' diff --git a/apps/sim/lib/billing/webhooks/outbox-events.ts b/apps/sim/lib/billing/webhooks/outbox-events.ts new file mode 100644 index 00000000000..5a7b53307c5 --- /dev/null +++ b/apps/sim/lib/billing/webhooks/outbox-events.ts @@ -0,0 +1,23 @@ +export const OUTBOX_EVENT_TYPES = { + /** + * Sync a subscription's `cancel_at_period_end` flag from our DB to + * Stripe. The handler reads the current DB value at processing time + * — so rapid cancel→uncancel→cancel sequences always converge on + * the last-committed DB state regardless of outbox ordering. Callers + * enqueue this event after every DB change to `cancelAtPeriodEnd`. + */ + STRIPE_SYNC_CANCEL_AT_PERIOD_END: 'stripe.sync-cancel-at-period-end', + /** Cancel in Stripe; the verified deletion webhook remains the only DB entitlement authority. */ + STRIPE_CANCEL_SUBSCRIPTION_IMMEDIATELY: 'stripe.cancel-subscription-immediately', + /** + * Sync a Team subscription's price and seat quantity from our DB to + * Stripe. The handler reads the current DB plan + seats at processing + * time and reconciles the Stripe item's price (e.g. after a Pro→Team + * conversion) and quantity, charging the proration via `always_invoice`. + * A failed charge surfaces through Stripe dunning and the existing + * billing-blocked system, never under the synchronous accept path. + */ + STRIPE_SYNC_SUBSCRIPTION_SEATS: 'stripe.sync-subscription-seats', + STRIPE_THRESHOLD_OVERAGE_INVOICE: 'stripe.threshold-overage-invoice', + STRIPE_SYNC_CUSTOMER_CONTACT: 'stripe.sync-customer-contact', +} as const diff --git a/apps/sim/lib/billing/webhooks/outbox-handlers.test.ts b/apps/sim/lib/billing/webhooks/outbox-handlers.test.ts index 207d17fbeec..b93683292eb 100644 --- a/apps/sim/lib/billing/webhooks/outbox-handlers.test.ts +++ b/apps/sim/lib/billing/webhooks/outbox-handlers.test.ts @@ -16,7 +16,8 @@ vi.mock('@/lib/billing/plans', () => ({ vi.mock('@/lib/billing/stripe-payment-method', () => stripePaymentMethodMock) -import { billingOutboxHandlers, OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' +import { billingOutboxHandlers } from '@/lib/billing/webhooks/outbox-handlers' const stripeMock = { subscriptions: { diff --git a/apps/sim/lib/billing/webhooks/outbox-handlers.ts b/apps/sim/lib/billing/webhooks/outbox-handlers.ts index 58613b41ca5..1d3aa61109e 100644 --- a/apps/sim/lib/billing/webhooks/outbox-handlers.ts +++ b/apps/sim/lib/billing/webhooks/outbox-handlers.ts @@ -9,34 +9,11 @@ import { getPlanByName } from '@/lib/billing/plans' import { requireStripeClient } from '@/lib/billing/stripe-client' import { resolveDefaultPaymentMethod } from '@/lib/billing/stripe-payment-method' import { hasPaidSubscriptionStatus } from '@/lib/billing/subscriptions/utils' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' import type { OutboxHandler } from '@/lib/core/outbox/service' const logger = createLogger('BillingOutboxHandlers') -export const OUTBOX_EVENT_TYPES = { - /** - * Sync a subscription's `cancel_at_period_end` flag from our DB to - * Stripe. The handler reads the current DB value at processing time - * — so rapid cancel→uncancel→cancel sequences always converge on - * the last-committed DB state regardless of outbox ordering. Callers - * enqueue this event after every DB change to `cancelAtPeriodEnd`. - */ - STRIPE_SYNC_CANCEL_AT_PERIOD_END: 'stripe.sync-cancel-at-period-end', - /** Cancel in Stripe; the verified deletion webhook remains the only DB entitlement authority. */ - STRIPE_CANCEL_SUBSCRIPTION_IMMEDIATELY: 'stripe.cancel-subscription-immediately', - /** - * Sync a Team subscription's price and seat quantity from our DB to - * Stripe. The handler reads the current DB plan + seats at processing - * time and reconciles the Stripe item's price (e.g. after a Pro→Team - * conversion) and quantity, charging the proration via `always_invoice`. - * A failed charge surfaces through Stripe dunning and the existing - * billing-blocked system, never under the synchronous accept path. - */ - STRIPE_SYNC_SUBSCRIPTION_SEATS: 'stripe.sync-subscription-seats', - STRIPE_THRESHOLD_OVERAGE_INVOICE: 'stripe.threshold-overage-invoice', - STRIPE_SYNC_CUSTOMER_CONTACT: 'stripe.sync-customer-contact', -} as const - interface StripeSyncCancelAtPeriodEndPayload { stripeSubscriptionId: string /** The DB subscription row id — also our source-of-truth pointer. */ diff --git a/apps/sim/lib/core/outbox/handlers.test.ts b/apps/sim/lib/core/outbox/handlers.test.ts new file mode 100644 index 00000000000..fe04a7414f0 --- /dev/null +++ b/apps/sim/lib/core/outbox/handlers.test.ts @@ -0,0 +1,65 @@ +import { readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { OUTBOX_HANDLER_GROUPS } from '@/lib/core/outbox/handlers' +import type { OutboxHandlerRegistry } from '@/lib/core/outbox/service' + +const APP_ROOT = fileURLToPath(new URL('../../..', import.meta.url)) +const HANDLER_MAP_EXPORT = /^export const (\w+OutboxHandlers)\b/gm +const SKIPPED_DIRECTORIES = new Set(['node_modules', 'public']) + +/** Every non-test source file under the app that exports a `OutboxHandlers` map. */ +function findHandlerMapExports(): { file: string; exportName: string }[] { + const found: { file: string; exportName: string }[] = [] + const walk = (directory: string) => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.name.startsWith('.') || SKIPPED_DIRECTORIES.has(entry.name)) continue + const fullPath = path.join(directory, entry.name) + if (entry.isDirectory()) { + walk(fullPath) + continue + } + if (!/\.tsx?$/.test(entry.name) || /\.(test|integration)\.tsx?$/.test(entry.name)) continue + for (const [, exportName] of readFileSync(fullPath, 'utf8').matchAll(HANDLER_MAP_EXPORT)) { + found.push({ file: fullPath, exportName }) + } + } + } + walk(APP_ROOT) + return found +} + +describe('OUTBOX_HANDLER_GROUPS', () => { + it('routes every event type to a single handler module', () => { + const eventTypes = OUTBOX_HANDLER_GROUPS.flatMap((group) => group.events) + expect(new Set(eventTypes).size).toBe(eventTypes.length) + }) + + it.each(OUTBOX_HANDLER_GROUPS.map((group) => [group.events.join(', '), group] as const))( + 'declares exactly the event types its module handles: %s', + async (_name, group) => { + const handlers = await group.load() + expect(Object.keys(handlers).sort()).toEqual([...group.events].sort()) + }, + 60_000 + ) + + it('declares every event type of every exported handler map', async () => { + const handlerMaps = findHandlerMapExports() + expect(handlerMaps.length).toBeGreaterThan(0) + + const declared = new Set(OUTBOX_HANDLER_GROUPS.flatMap((group) => group.events)) + const undeclared: string[] = [] + for (const { file, exportName } of handlerMaps) { + const module: Record = await import(file) + const handlers = module[exportName] as OutboxHandlerRegistry + for (const eventType of Object.keys(handlers)) { + if (!declared.has(eventType)) { + undeclared.push(`${path.relative(APP_ROOT, file)}#${exportName}: ${eventType}`) + } + } + } + expect(undeclared).toEqual([]) + }, 60_000) +}) diff --git a/apps/sim/lib/core/outbox/handlers.ts b/apps/sim/lib/core/outbox/handlers.ts new file mode 100644 index 00000000000..3860af6500b --- /dev/null +++ b/apps/sim/lib/core/outbox/handlers.ts @@ -0,0 +1,240 @@ +import { ADMIN_INVITATION_OPERATION_EVENT_TYPE } from '@/lib/admin/invitation-operation-event' +import { ADMIN_MEMBER_OPERATION_EVENT_TYPE } from '@/lib/admin/member-operation-event' +import { + ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE, + ENTERPRISE_MEMBER_RECONCILIATION_EVENT_TYPE, + ENTERPRISE_METADATA_SYNC_EVENT_TYPE, + ENTERPRISE_PROVISION_EVENT_TYPE, + ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE, +} from '@/lib/billing/enterprise-outbox' +import { + ENTERPRISE_OWNER_ACTIVATION_EVENT_TYPE, + ENTERPRISE_OWNER_CLAIM_EVENT_TYPE, +} from '@/lib/billing/enterprise-owner-claim-events' +import { MEMBER_BILLING_RECONCILIATION_EVENT_TYPE } from '@/lib/billing/organizations/membership-reconciliation-event' +import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events' +import type { LazyOutboxHandlerGroup } from '@/lib/core/outbox/service' +import { DIRECT_GRANT_EMAIL_EVENT_TYPE } from '@/lib/invitations/direct-grant-event' +import { SLACK_SEARCH_TURN_EVENT } from '@/lib/knowledge/application/slack-search/turn-event' +import { + KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, + KNOWLEDGE_CONNECTOR_DETACH_EVENT, +} from '@/lib/knowledge/connectors/outbox-events' +import { + EMBEDDING_CHECKPOINT_CLEANUP_EVENT, + OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, +} from '@/lib/knowledge/documents/checkpoint-events' +import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-event' +import { + KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT, + KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, +} from '@/lib/knowledge/documents/processing-outbox-event' +import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup' +import { INBOX_CLEANUP_EVENT } from '@/lib/mothership/inbox/cleanup-event' +import { ORGANIZATION_RESOURCE_CLEANUP_EVENT } from '@/lib/organizations/resource-cleanup-event' +import { + WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, + WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT, +} from '@/lib/uploads/contexts/workspace/file-outbox-events' +import { WORKFLOW_DEPLOYMENT_OUTBOX_EVENTS } from '@/lib/workflows/deployment-outbox-events' +import { MIGRATED_INVITATION_EMAIL_EVENT_TYPE } from '@/lib/workspaces/admin-move-event' +import { + WORKSPACE_MCP_CHANGED_EVENT, + WORKSPACE_OPERATION_OBSERVE_EVENT, + WORKSPACE_WORKFLOWS_CHANGED_EVENT, +} from '@/lib/workspaces/operations/outbox-events' +import { + PERMISSION_ACCESS_REQUEST_CREATED_EVENT, + PERMISSION_ACCESS_REQUEST_DECIDED_EVENT, + PERMISSION_ACCESS_REQUEST_NOTIFY_ADMIN_EVENT, +} from '@/ee/access-requests/lib/notification-events' +import { FORK_CONTENT_COPY_EVENT } from '@/ee/workspace-forking/application/content-outbox-event' + +/** + * Every handler module the outbox processor serves, with the event types its handler map + * registers. A module is imported only when one of its event types is due, so a run never loads + * the dependencies of handlers it will not call. Event types come from dependency-free modules + * (or ones the processor already loads), so naming them here costs nothing. + * + * Each group's `events` must equal its loaded map's keys: a type missing here is never served and + * dead-letters as unhandled (`handlers.test.ts` enforces this). Each `load` destructures its + * import so the unused-export audit can see which export it reads; `(await import(x)).y` marks + * every export of the module as used. + */ +export const OUTBOX_HANDLER_GROUPS: readonly LazyOutboxHandlerGroup[] = [ + { + events: [SLACK_SEARCH_TURN_EVENT], + load: async () => { + const { slackSearchOutboxHandlers } = await import( + '@/lib/knowledge/application/slack-search/outbox' + ) + return slackSearchOutboxHandlers + }, + }, + { + events: [ADMIN_INVITATION_OPERATION_EVENT_TYPE], + load: async () => { + const { adminInvitationOperationOutboxHandlers } = await import( + '@/lib/admin/invitation-operation' + ) + return adminInvitationOperationOutboxHandlers + }, + }, + { + events: [ADMIN_MEMBER_OPERATION_EVENT_TYPE], + load: async () => { + const { adminMemberOperationOutboxHandlers } = await import('@/lib/admin/member-operation') + return adminMemberOperationOutboxHandlers + }, + }, + { + events: Object.values(OUTBOX_EVENT_TYPES), + load: async () => { + const { billingOutboxHandlers } = await import('@/lib/billing/webhooks/outbox-handlers') + return billingOutboxHandlers + }, + }, + { + events: [MEMBER_BILLING_RECONCILIATION_EVENT_TYPE], + load: async () => { + const { membershipBillingOutboxHandlers } = await import( + '@/lib/billing/organizations/membership-reconciliation' + ) + return membershipBillingOutboxHandlers + }, + }, + { + events: [ + ENTERPRISE_PROVISION_EVENT_TYPE, + ENTERPRISE_METADATA_SYNC_EVENT_TYPE, + ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE, + ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE, + ENTERPRISE_MEMBER_RECONCILIATION_EVENT_TYPE, + ], + load: async () => { + const { enterpriseIssuanceOutboxHandlers } = await import( + '@/lib/billing/enterprise-provisioning' + ) + return enterpriseIssuanceOutboxHandlers + }, + }, + { + events: [ENTERPRISE_OWNER_CLAIM_EVENT_TYPE, ENTERPRISE_OWNER_ACTIVATION_EVENT_TYPE], + load: async () => { + const { enterpriseOwnerClaimOutboxHandlers } = await import( + '@/lib/billing/enterprise-owner-claim' + ) + return enterpriseOwnerClaimOutboxHandlers + }, + }, + { + events: [MIGRATED_INVITATION_EMAIL_EVENT_TYPE], + load: async () => { + const { invitationMigrationOutboxHandlers } = await import('@/lib/workspaces/admin-move') + return invitationMigrationOutboxHandlers + }, + }, + { + events: [DIRECT_GRANT_EMAIL_EVENT_TYPE], + load: async () => { + const { directGrantOutboxHandlers } = await import('@/lib/invitations/direct-grant') + return directGrantOutboxHandlers + }, + }, + { + events: [ + KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, + KNOWLEDGE_CONNECTOR_DETACH_EVENT, + KNOWLEDGE_STORAGE_CLEANUP_EVENT, + OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, + KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT, + EMBEDDING_CHECKPOINT_CLEANUP_EVENT, + KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, + KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT, + KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT, + ], + load: async () => { + const { knowledgeDocumentProcessingOutboxHandlers } = await import( + '@/lib/knowledge/documents/processing-outbox-handler' + ) + return knowledgeDocumentProcessingOutboxHandlers + }, + }, + { + events: [ORGANIZATION_RESOURCE_CLEANUP_EVENT], + load: async () => { + const { organizationResourceCleanupOutboxHandlers } = await import( + '@/lib/organizations/resource-cleanup' + ) + return organizationResourceCleanupOutboxHandlers + }, + }, + { + events: [INBOX_CLEANUP_EVENT], + load: async () => { + const { inboxCleanupOutboxHandlers } = await import('@/lib/mothership/inbox/cleanup-outbox') + return inboxCleanupOutboxHandlers + }, + }, + { + events: [ + PERMISSION_ACCESS_REQUEST_CREATED_EVENT, + PERMISSION_ACCESS_REQUEST_NOTIFY_ADMIN_EVENT, + PERMISSION_ACCESS_REQUEST_DECIDED_EVENT, + ], + load: async () => { + const { permissionAccessRequestOutboxHandlers } = await import( + '@/ee/access-requests/lib/notifications' + ) + return permissionAccessRequestOutboxHandlers + }, + }, + { + events: [WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT], + load: async () => { + const { workspaceFileLiveDocOutboxHandlers } = await import( + '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' + ) + return workspaceFileLiveDocOutboxHandlers + }, + }, + { + events: [WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT], + load: async () => { + const { workspaceFileStorageCleanupOutboxHandlers } = await import( + '@/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox' + ) + return workspaceFileStorageCleanupOutboxHandlers + }, + }, + { + events: Object.values(WORKFLOW_DEPLOYMENT_OUTBOX_EVENTS), + load: async () => { + const { workflowDeploymentOutboxHandlers } = await import('@/lib/workflows/deployment-outbox') + return workflowDeploymentOutboxHandlers + }, + }, + { + events: [ + WORKSPACE_MCP_CHANGED_EVENT, + WORKSPACE_OPERATION_OBSERVE_EVENT, + WORKSPACE_WORKFLOWS_CHANGED_EVENT, + ], + load: async () => { + const { workspaceOperationOutboxHandlers } = await import( + '@/lib/workspaces/operations/outbox' + ) + return workspaceOperationOutboxHandlers + }, + }, + { + events: [FORK_CONTENT_COPY_EVENT], + load: async () => { + const { forkContentOutboxHandlers } = await import( + '@/ee/workspace-forking/application/content-outbox' + ) + return forkContentOutboxHandlers + }, + }, +] diff --git a/apps/sim/lib/core/outbox/processor.test.ts b/apps/sim/lib/core/outbox/processor.test.ts index a8c8d0b26b2..c818129df29 100644 --- a/apps/sim/lib/core/outbox/processor.test.ts +++ b/apps/sim/lib/core/outbox/processor.test.ts @@ -1,4 +1,3 @@ -import { billingOutboxHandlersMock } from '@sim/testing/mocks/billing-outbox-handlers.mock' import { outboxServiceMock, outboxServiceMockFns } from '@sim/testing/mocks/outbox-service.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -18,40 +17,7 @@ vi.mock('@/ee/workspace-forking/lib/background-work/store', () => ({ vi.mock('@/lib/knowledge/connectors/connector-error', () => ({ getConnectorFailureDiagnostic: () => undefined, })) -vi.mock('@/lib/admin/invitation-operation', () => ({ adminInvitationOperationOutboxHandlers: {} })) -vi.mock('@/lib/admin/member-operation', () => ({ adminMemberOperationOutboxHandlers: {} })) -vi.mock('@/lib/billing/enterprise-owner-claim', () => ({ enterpriseOwnerClaimOutboxHandlers: {} })) -vi.mock('@/lib/billing/enterprise-provisioning', () => ({ enterpriseIssuanceOutboxHandlers: {} })) -vi.mock('@/lib/billing/organizations/membership-reconciliation', () => ({ - membershipBillingOutboxHandlers: {}, -})) -vi.mock('@/lib/billing/webhooks/outbox-handlers', () => billingOutboxHandlersMock) -vi.mock('@/lib/invitations/direct-grant', () => ({ directGrantOutboxHandlers: {} })) -vi.mock('@/lib/knowledge/application/slack-search/outbox', () => ({ - slackSearchOutboxHandlers: {}, -})) -vi.mock('@/lib/knowledge/documents/processing-outbox-handler', () => ({ - knowledgeDocumentProcessingOutboxHandlers: {}, -})) -vi.mock('@/lib/mothership/inbox/cleanup-outbox', () => ({ inboxCleanupOutboxHandlers: {} })) -vi.mock('@/lib/organizations/resource-cleanup', () => ({ - organizationResourceCleanupOutboxHandlers: {}, -})) -vi.mock('@/ee/access-requests/lib/notifications', () => ({ - permissionAccessRequestOutboxHandlers: {}, -})) -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox', () => ({ - workspaceFileLiveDocOutboxHandlers: {}, -})) -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox', () => ({ - workspaceFileStorageCleanupOutboxHandlers: {}, -})) -vi.mock('@/lib/workflows/deployment-outbox', () => ({ workflowDeploymentOutboxHandlers: {} })) -vi.mock('@/lib/workspaces/admin-move', () => ({ invitationMigrationOutboxHandlers: {} })) -vi.mock('@/lib/workspaces/operations/outbox', () => ({ workspaceOperationOutboxHandlers: {} })) -vi.mock('@/ee/workspace-forking/application/content-outbox', () => ({ - forkContentOutboxHandlers: {}, -})) +vi.mock('@/lib/core/outbox/handlers', () => ({ OUTBOX_HANDLER_GROUPS: [] })) import { runOutboxProcessor } from '@/lib/core/outbox/processor' diff --git a/apps/sim/lib/core/outbox/processor.ts b/apps/sim/lib/core/outbox/processor.ts index 1d1e2e1da5e..37d27145a1e 100644 --- a/apps/sim/lib/core/outbox/processor.ts +++ b/apps/sim/lib/core/outbox/processor.ts @@ -1,58 +1,20 @@ import { db } from '@sim/db' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { adminInvitationOperationOutboxHandlers } from '@/lib/admin/invitation-operation' -import { adminMemberOperationOutboxHandlers } from '@/lib/admin/member-operation' -import { enterpriseOwnerClaimOutboxHandlers } from '@/lib/billing/enterprise-owner-claim' -import { enterpriseIssuanceOutboxHandlers } from '@/lib/billing/enterprise-provisioning' -import { membershipBillingOutboxHandlers } from '@/lib/billing/organizations/membership-reconciliation' -import { billingOutboxHandlers } from '@/lib/billing/webhooks/outbox-handlers' import { OUTBOX_PROCESSOR_MAX_RUNTIME_MS, OUTBOX_PROCESSOR_RECOVERY_CUTOFF_MS, } from '@/lib/core/outbox/constants' +import { OUTBOX_HANDLER_GROUPS } from '@/lib/core/outbox/handlers' import { pruneCompletedOutboxEvents } from '@/lib/core/outbox/retention' import { type ProcessOutboxResult, processOutboxEvents } from '@/lib/core/outbox/service' import { DeadlineExceededError } from '@/lib/core/utils/deadline' -import { directGrantOutboxHandlers } from '@/lib/invitations/direct-grant' -import { slackSearchOutboxHandlers } from '@/lib/knowledge/application/slack-search/outbox' import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' -import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { recoverKnowledgeDocumentProcessing } from '@/lib/knowledge/documents/processing-recovery' -import { inboxCleanupOutboxHandlers } from '@/lib/mothership/inbox/cleanup-outbox' -import { organizationResourceCleanupOutboxHandlers } from '@/lib/organizations/resource-cleanup' -import { workspaceFileLiveDocOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' -import { workspaceFileStorageCleanupOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox' -import { workflowDeploymentOutboxHandlers } from '@/lib/workflows/deployment-outbox' -import { invitationMigrationOutboxHandlers } from '@/lib/workspaces/admin-move' -import { workspaceOperationOutboxHandlers } from '@/lib/workspaces/operations/outbox' -import { permissionAccessRequestOutboxHandlers } from '@/ee/access-requests/lib/notifications' -import { forkContentOutboxHandlers } from '@/ee/workspace-forking/application/content-outbox' import { reapStaleBackgroundWork } from '@/ee/workspace-forking/lib/background-work/store' const logger = createLogger('OutboxProcessor') -const handlers = { - ...slackSearchOutboxHandlers, - ...adminInvitationOperationOutboxHandlers, - ...adminMemberOperationOutboxHandlers, - ...billingOutboxHandlers, - ...membershipBillingOutboxHandlers, - ...enterpriseIssuanceOutboxHandlers, - ...enterpriseOwnerClaimOutboxHandlers, - ...invitationMigrationOutboxHandlers, - ...directGrantOutboxHandlers, - ...knowledgeDocumentProcessingOutboxHandlers, - ...organizationResourceCleanupOutboxHandlers, - ...inboxCleanupOutboxHandlers, - ...permissionAccessRequestOutboxHandlers, - ...workspaceFileLiveDocOutboxHandlers, - ...workspaceFileStorageCleanupOutboxHandlers, - ...workflowDeploymentOutboxHandlers, - ...workspaceOperationOutboxHandlers, - ...forkContentOutboxHandlers, -} as const - export interface OutboxProcessorResult { result: ProcessOutboxResult recoveredDocuments: number @@ -63,7 +25,7 @@ export interface OutboxProcessorResult { /** Processes one bounded batch and its recovery work in either the worker or self-hosted cron. */ export async function runOutboxProcessor(): Promise { const startedAt = Date.now() - const result = await processOutboxEvents(handlers, { + const result = await processOutboxEvents(OUTBOX_HANDLER_GROUPS, { batchSize: 500, maxRuntimeMs: OUTBOX_PROCESSOR_MAX_RUNTIME_MS, minRemainingMs: 95_000, diff --git a/apps/sim/lib/core/outbox/service.test.ts b/apps/sim/lib/core/outbox/service.test.ts index e6934d46e80..692a93f482f 100644 --- a/apps/sim/lib/core/outbox/service.test.ts +++ b/apps/sim/lib/core/outbox/service.test.ts @@ -2,6 +2,7 @@ import { outboxEvent } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { dbChainMock, dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { idMock, idMockFns } from '@sim/testing/mocks/id.mock' +import { eq } from 'drizzle-orm' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' type OutboxRow = { @@ -573,3 +574,101 @@ describe('processOutboxEvents — reaper recovery', () => { expect(reaperUpdate?.lockedAt).toBeNull() }) }) + +describe('processOutboxEvents — lazy handler groups', () => { + beforeEach(() => { + resetDbChainMock() + }) + + /** Event types the claim phase selected, in claim order. */ + const claimedEventTypes = (): unknown[] => + vi + .mocked(eq) + .mock.calls.filter(([column]) => column === outboxEvent.eventType) + .map(([, eventType]) => eventType) + + it('imports only the handler modules whose event types are due', async () => { + const handler = vi.fn(async () => {}) + const idle = { events: ['test.idle'], load: vi.fn(async () => ({ 'test.idle': vi.fn() })) } + queuePendingEvents([makePendingRow()]) + holdLease() + + const result = await processOutboxEvents([ + { events: ['test.event'], load: async () => ({ 'test.event': handler }) }, + idle, + ]) + + expect(result.processed).toBe(1) + expect(handler).toHaveBeenCalledOnce() + expect(idle.load).not.toHaveBeenCalled() + }) + + it('leaves the events of a module that fails to import unclaimed and imports it again next run', async () => { + const recoveredHandler = vi.fn(async () => {}) + const healthyHandler = vi.fn(async () => {}) + const flaky = { + events: ['test.flaky'], + load: vi + .fn() + .mockRejectedValueOnce(new Error('Cannot find module')) + .mockResolvedValue({ 'test.flaky': recoveredHandler }), + } + const groups = [ + flaky, + { events: ['test.healthy'], load: async () => ({ 'test.healthy': healthyHandler }) }, + ] + dbChainMockFns.execute.mockResolvedValueOnce([ + { eventType: 'test.flaky' }, + { eventType: 'test.healthy' }, + ]) + queueTableRows(outboxEvent, [makePendingRow({ eventType: 'test.healthy' })]) + holdLease() + + expect(await processOutboxEvents(groups)).toMatchObject({ + processed: 1, + retried: 0, + deadLettered: 0, + }) + expect(healthyHandler).toHaveBeenCalledOnce() + expect(claimedEventTypes()).not.toContain('test.flaky') + expect(updateSets().some((set) => 'attempts' in set)).toBe(false) + + resetDbChainMock() + queuePendingEvents([makePendingRow({ eventType: 'test.flaky' })]) + holdLease() + + expect(await processOutboxEvents(groups)).toMatchObject({ processed: 1 }) + expect(recoveredHandler).toHaveBeenCalledOnce() + }) + + it('sizes the pre-claim deadline check from the imported handler window', async () => { + const handler = withOutboxHandlerTimeout( + vi.fn(async () => {}), + 550_000 + ) + queuePendingEvents([makePendingRow()]) + holdLease() + + const result = await processOutboxEvents( + [{ events: ['test.event'], load: async () => ({ 'test.event': handler }) }], + { maxRuntimeMs: 110_000 } + ) + + expect(result).toMatchObject({ processed: 0, retried: 0 }) + expect(handler).not.toHaveBeenCalled() + expect(updateSets().some((set) => set.status === 'processing')).toBe(false) + }) + + it('still retries an event type that no handler module declares', async () => { + const load = vi.fn(async () => ({ 'test.event': vi.fn() })) + queuePendingEvents([makePendingRow({ eventType: 'unknown.event' })]) + holdLease() + + const result = await processOutboxEvents([{ events: ['test.event'], load }]) + + expect(result.retried).toBe(1) + const retry = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) + expect(retry?.attempts).toBe(1) + expect(load).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/core/outbox/service.ts b/apps/sim/lib/core/outbox/service.ts index 472aa524232..0591d1cabe3 100644 --- a/apps/sim/lib/core/outbox/service.ts +++ b/apps/sim/lib/core/outbox/service.ts @@ -144,6 +144,23 @@ export function withOutboxHandlerTimeout( */ export type OutboxHandlerRegistry = Record +/** + * A handler module imported only when one of its event types is due. `events` lists exactly + * the keys of the registry `load` resolves to; a type absent from `events` is never served. + */ +export interface LazyOutboxHandlerGroup { + readonly events: readonly string[] + readonly load: () => Promise +} + +type OutboxHandlerSource = OutboxHandlerRegistry | readonly LazyOutboxHandlerGroup[] + +function isLazyOutboxHandlerSource( + source: OutboxHandlerSource +): source is readonly LazyOutboxHandlerGroup[] { + return Array.isArray(source) +} + export interface EnqueueOptions { /** Caller-owned idempotency key. Defaults to a generated UUID. */ id?: string @@ -427,9 +444,12 @@ export async function hasInflightOutboxEvent( * bulk maintenance cannot monopolize delivery. Each type serves its earliest * available events first. Safe to call concurrently from multiple workers — * `SELECT FOR UPDATE SKIP LOCKED` serializes claims. + * + * Lazy handler groups are imported for the ready event types before any claim, + * so import time never runs inside a handler's window or a claimed lease. */ export async function processOutboxEvents( - handlers: OutboxHandlerRegistry, + handlerSource: OutboxHandlerSource, options: { batchSize?: number; maxRuntimeMs?: number; minRemainingMs?: number } = {} ): Promise { const startedAt = Date.now() @@ -447,7 +467,14 @@ export async function processOutboxEvents( reaped = await reapStuckProcessingRows() phase = 'discover' const readyTypes = await db.execute<{ eventType: string }>(readyEventTypesQuery(new Date())) - const eligibleTypes = readyTypes.map(({ eventType }) => eventType) + const readyEventTypes = readyTypes.map(({ eventType }) => eventType) + const { handlers, unavailableEventTypes } = await resolveOutboxHandlers( + handlerSource, + readyEventTypes + ) + const eligibleTypes = readyEventTypes.filter( + (eventType) => !unavailableEventTypes.has(eventType) + ) let cursor = 0 let claimed = 0 @@ -500,6 +527,41 @@ export async function processOutboxEvents( } } +/** + * Imports the lazy groups that serve any ready event type. A group whose import fails leaves its + * event types unclaimed for this run: unlike a missing handler, which spends an attempt and + * eventually dead-letters, a failed import says nothing about the events, so they stay pending + * for a later run. Event types outside every group still reach the missing-handler path. + */ +async function resolveOutboxHandlers( + source: OutboxHandlerSource, + readyEventTypes: readonly string[] +): Promise<{ handlers: OutboxHandlerRegistry; unavailableEventTypes: Set }> { + const unavailableEventTypes = new Set() + if (!isLazyOutboxHandlerSource(source)) return { handlers: source, unavailableEventTypes } + + const ready = new Set(readyEventTypes) + const dueGroups = source.filter((group) => group.events.some((eventType) => ready.has(eventType))) + const loaded = await Promise.allSettled(dueGroups.map((group) => group.load())) + const handlers: OutboxHandlerRegistry = {} + for (const [index, outcome] of loaded.entries()) { + const { events } = dueGroups[index] + if (outcome.status === 'rejected') { + for (const eventType of events) unavailableEventTypes.add(eventType) + logger.error('Outbox handler module failed to load; leaving its events pending', { + eventTypes: events, + error: describeError(outcome.reason), + }) + continue + } + for (const eventType of events) { + const handler = outcome.value[eventType] + if (handler) handlers[eventType] = handler + } + } + return { handlers, unavailableEventTypes } +} + /** * Process a specific outbox event immediately after its surrounding * transaction commits. Safe to race with the cron worker: the claim uses diff --git a/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts index e23caca3fde..777bf13656c 100644 --- a/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts @@ -40,8 +40,8 @@ import { import { searchKnowledge } from '@/lib/knowledge/application/search' import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' import { addDocument, persistDocumentAcls } from '@/lib/knowledge/connectors/sync-persistence' +import { EMBEDDING_CHECKPOINT_CLEANUP_EVENT } from '@/lib/knowledge/documents/checkpoint-events' import * as embeddingCheckpoints from '@/lib/knowledge/documents/embedding-checkpoints' -import { EMBEDDING_CHECKPOINT_CLEANUP_EVENT } from '@/lib/knowledge/documents/embedding-checkpoints' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { assertDocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' import { processDocumentsWithQueue } from '@/lib/knowledge/documents/service' diff --git a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts index a3b567598f1..eb1cb08c663 100644 --- a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts @@ -51,7 +51,7 @@ import { } from '@/lib/knowledge/connectors/member-observations' import { createContentSyncLease, createMemberSyncLease } from '@/lib/knowledge/connectors/sync-lock' import { addDocument } from '@/lib/knowledge/connectors/sync-persistence' -import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-dispatch' +import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-event' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { assertDocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' import * as providerContinuation from '@/lib/knowledge/documents/processing-provider-continuation' diff --git a/apps/sim/lib/knowledge/__integration__/purged-detach-reservation.integration.ts b/apps/sim/lib/knowledge/__integration__/purged-detach-reservation.integration.ts index 88d9c24e09d..ad5fe38601e 100644 --- a/apps/sim/lib/knowledge/__integration__/purged-detach-reservation.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/purged-detach-reservation.integration.ts @@ -23,10 +23,8 @@ import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { - KNOWLEDGE_CONNECTOR_DETACH_EVENT, - settleDetachedConnectorReservations, -} from '@/lib/knowledge/connectors/detachment' +import { settleDetachedConnectorReservations } from '@/lib/knowledge/connectors/detachment' +import { KNOWLEDGE_CONNECTOR_DETACH_EVENT } from '@/lib/knowledge/connectors/outbox-events' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { createSingleDocument, hardDeleteDocuments } from '@/lib/knowledge/documents/service' import { performDeleteKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' diff --git a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts index e4580b084e3..bf1a715cc08 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts @@ -30,7 +30,7 @@ import { readKnowledgeDocument, updateKnowledgeDocument, } from '@/lib/knowledge/application/documents' -import { KNOWLEDGE_CONNECTOR_DETACH_EVENT } from '@/lib/knowledge/connectors/detachment' +import { KNOWLEDGE_CONNECTOR_DETACH_EVENT } from '@/lib/knowledge/connectors/outbox-events' import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' import { persistSkippedDocuments } from '@/lib/knowledge/connectors/sync-persistence' import * as documentProcessor from '@/lib/knowledge/documents/document-processor' diff --git a/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts b/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts index 41d9e626199..7cb1651e63c 100644 --- a/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/storage-accounting.integration.ts @@ -26,8 +26,10 @@ import { } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' -import { KNOWLEDGE_CONNECTOR_CLEANUP_EVENT } from '@/lib/knowledge/connectors/deletion' -import { KNOWLEDGE_CONNECTOR_DETACH_EVENT } from '@/lib/knowledge/connectors/detachment' +import { + KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, + KNOWLEDGE_CONNECTOR_DETACH_EVENT, +} from '@/lib/knowledge/connectors/outbox-events' import { createContentSyncLease, SyncLockLostException } from '@/lib/knowledge/connectors/sync-lock' import { persistSkippedDocuments } from '@/lib/knowledge/connectors/sync-persistence' import { diff --git a/apps/sim/lib/knowledge/application/slack-search/outbox.ts b/apps/sim/lib/knowledge/application/slack-search/outbox.ts index e021ae70450..bdb38b6f730 100644 --- a/apps/sim/lib/knowledge/application/slack-search/outbox.ts +++ b/apps/sim/lib/knowledge/application/slack-search/outbox.ts @@ -7,9 +7,9 @@ import { type OutboxHandlerRegistry, processOutboxEventById, } from '@/lib/core/outbox/service' +import { SLACK_SEARCH_TURN_EVENT } from '@/lib/knowledge/application/slack-search/turn-event' import { readySlackSearchDispatches, - SLACK_SEARCH_TURN_EVENT, slackSearchTurnOutboxId, } from '@/lib/knowledge/application/slack-search/turns' import { enqueueSlackSearch } from '@/lib/slack-search/queue' diff --git a/apps/sim/lib/knowledge/application/slack-search/turn-event.ts b/apps/sim/lib/knowledge/application/slack-search/turn-event.ts new file mode 100644 index 00000000000..de13ed266fd --- /dev/null +++ b/apps/sim/lib/knowledge/application/slack-search/turn-event.ts @@ -0,0 +1 @@ +export const SLACK_SEARCH_TURN_EVENT = 'slack-search.turn' diff --git a/apps/sim/lib/knowledge/application/slack-search/turns.ts b/apps/sim/lib/knowledge/application/slack-search/turns.ts index 4adaf86dbb3..e2385bbcfe3 100644 --- a/apps/sim/lib/knowledge/application/slack-search/turns.ts +++ b/apps/sim/lib/knowledge/application/slack-search/turns.ts @@ -9,6 +9,7 @@ import { requireSlackSearchConversationSender, resolveSlackSearchChatRecord, } from '@/lib/knowledge/application/slack-search/chat' +import { SLACK_SEARCH_TURN_EVENT } from '@/lib/knowledge/application/slack-search/turn-event' import { SLACK_SEARCH_CONCURRENCY, SLACK_SEARCH_MAX_DURATION_SECONDS, @@ -21,7 +22,6 @@ import { } from '@/lib/slack-search/conversation' import { type SlackSearchJob, slackSearchJobSchema } from '@/lib/slack-search/types' -export const SLACK_SEARCH_TURN_EVENT = 'slack-search.turn' export function slackSearchTurnOutboxId(turnId: string) { return `slack-search-turn:${turnId}` } diff --git a/apps/sim/lib/knowledge/connectors/deletion.test.ts b/apps/sim/lib/knowledge/connectors/deletion.test.ts index 931c75d1269..ebd4ca3c5d8 100644 --- a/apps/sim/lib/knowledge/connectors/deletion.test.ts +++ b/apps/sim/lib/knowledge/connectors/deletion.test.ts @@ -18,10 +18,8 @@ vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ vi.mock('@/lib/knowledge/tags/service', () => knowledgeTagsServiceMock) vi.mock('@/lib/knowledge/connectors/member-access', () => knowledgeMemberAccessMock) -import { - cleanupKnowledgeConnector, - KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, -} from '@/lib/knowledge/connectors/deletion' +import { cleanupKnowledgeConnector } from '@/lib/knowledge/connectors/deletion' +import { KNOWLEDGE_CONNECTOR_CLEANUP_EVENT } from '@/lib/knowledge/connectors/outbox-events' const mocks = { ...hoisted, diff --git a/apps/sim/lib/knowledge/connectors/deletion.ts b/apps/sim/lib/knowledge/connectors/deletion.ts index ab08f2b5f7f..a8105052f26 100644 --- a/apps/sim/lib/knowledge/connectors/deletion.ts +++ b/apps/sim/lib/knowledge/connectors/deletion.ts @@ -17,10 +17,10 @@ import { } from '@/lib/core/outbox/service' import type { DbOrTx } from '@/lib/db/types' import { revokeKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' +import { KNOWLEDGE_CONNECTOR_CLEANUP_EVENT } from '@/lib/knowledge/connectors/outbox-events' import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup' import { cleanupUnusedTagDefinitions } from '@/lib/knowledge/tags/service' -export const KNOWLEDGE_CONNECTOR_CLEANUP_EVENT = 'knowledge.connector.cleanup' const DOCUMENT_BATCH_SIZE = 250 const EMBEDDING_BATCH_SIZE = 1_000 const RELATED_ROW_BATCH_SIZE = 1_000 diff --git a/apps/sim/lib/knowledge/connectors/detachment.test.ts b/apps/sim/lib/knowledge/connectors/detachment.test.ts index 25978157d12..6bfdecd7ca7 100644 --- a/apps/sim/lib/knowledge/connectors/detachment.test.ts +++ b/apps/sim/lib/knowledge/connectors/detachment.test.ts @@ -24,9 +24,9 @@ vi.mock('@/lib/knowledge/tags/service', () => knowledgeTagsServiceMock) import { detachKnowledgeConnector, - KNOWLEDGE_CONNECTOR_DETACH_EVENT, settleDetachedConnectorReservations, } from '@/lib/knowledge/connectors/detachment' +import { KNOWLEDGE_CONNECTOR_DETACH_EVENT } from '@/lib/knowledge/connectors/outbox-events' const mocks = { resolveStorage: billingStorageMockFns.mockResolveStorageBillingContext, diff --git a/apps/sim/lib/knowledge/connectors/detachment.ts b/apps/sim/lib/knowledge/connectors/detachment.ts index ef5985e50f4..357329767f6 100644 --- a/apps/sim/lib/knowledge/connectors/detachment.ts +++ b/apps/sim/lib/knowledge/connectors/detachment.ts @@ -25,8 +25,8 @@ import { import type { DbOrTx } from '@/lib/db/types' import { removeDrainedConnector } from '@/lib/knowledge/connectors/deletion' import { revokeKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' +import { KNOWLEDGE_CONNECTOR_DETACH_EVENT } from '@/lib/knowledge/connectors/outbox-events' -export const KNOWLEDGE_CONNECTOR_DETACH_EVENT = 'knowledge.connector.detach' const DOCUMENT_BATCH_SIZE = 100 const MAX_BATCHES_PER_RUN = 4 const RUN_BUDGET_MS = 30_000 diff --git a/apps/sim/lib/knowledge/connectors/outbox-events.ts b/apps/sim/lib/knowledge/connectors/outbox-events.ts new file mode 100644 index 00000000000..9127b4da134 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/outbox-events.ts @@ -0,0 +1,2 @@ +export const KNOWLEDGE_CONNECTOR_CLEANUP_EVENT = 'knowledge.connector.cleanup' +export const KNOWLEDGE_CONNECTOR_DETACH_EVENT = 'knowledge.connector.detach' diff --git a/apps/sim/lib/knowledge/documents/checkpoint-events.ts b/apps/sim/lib/knowledge/documents/checkpoint-events.ts new file mode 100644 index 00000000000..1ee39dc9551 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/checkpoint-events.ts @@ -0,0 +1,2 @@ +export const OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT = 'knowledge.document.ocr-checkpoint.expire' +export const EMBEDDING_CHECKPOINT_CLEANUP_EVENT = 'knowledge.document.embedding-checkpoint.expire' diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index e8068c08a7e..6bb0739a20c 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -75,7 +75,7 @@ import { PermanentDocumentProcessingError, UsageLimitDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' -import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-dispatch' +import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-event' import { KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' import { processDocumentAsync, processDocumentsWithQueue } from '@/lib/knowledge/documents/service' import { MAX_PROCESSING_ATTEMPTS } from '@/lib/knowledge/documents/types' diff --git a/apps/sim/lib/knowledge/documents/embedding-checkpoints.test.ts b/apps/sim/lib/knowledge/documents/embedding-checkpoints.test.ts index 49dbee708ca..c69d51e6696 100644 --- a/apps/sim/lib/knowledge/documents/embedding-checkpoints.test.ts +++ b/apps/sim/lib/knowledge/documents/embedding-checkpoints.test.ts @@ -6,10 +6,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) vi.mock('@/lib/embeddings/client', () => ({ EMBEDDING_RETRY_BUDGET_MS: 150000 })) +import { EMBEDDING_CHECKPOINT_CLEANUP_EVENT } from '@/lib/knowledge/documents/checkpoint-events' import { cleanupEmbeddingCheckpoint, createEmbeddingCheckpoints, - EMBEDDING_CHECKPOINT_CLEANUP_EVENT, } from '@/lib/knowledge/documents/embedding-checkpoints' const identity = { key: sha256Hex('request'), itemCount: 2, dimensions: 2 } diff --git a/apps/sim/lib/knowledge/documents/embedding-checkpoints.ts b/apps/sim/lib/knowledge/documents/embedding-checkpoints.ts index c3295ccaf1e..3f319760ade 100644 --- a/apps/sim/lib/knowledge/documents/embedding-checkpoints.ts +++ b/apps/sim/lib/knowledge/documents/embedding-checkpoints.ts @@ -7,6 +7,7 @@ import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider- import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { EMBEDDING_RETRY_BUDGET_MS } from '@/lib/embeddings/client' import type { EmbeddingBatchCheckpoints, EmbeddingBatchIdentity } from '@/lib/embeddings/types' +import { EMBEDDING_CHECKPOINT_CLEANUP_EVENT } from '@/lib/knowledge/documents/checkpoint-events' import { checkpointIo, isMissingCheckpointObject, @@ -23,7 +24,6 @@ const MAX_HEADER_BYTES = 1024 const MAX_VECTOR_BYTES = 16 * 1024 * 1024 const CLEANUP_WRITE_MARGIN_MS = 15 * 60 * 1000 const KEY_PATTERN = /^knowledge-embedding-checkpoints\/v1\/[a-f0-9]{64}\/[a-f0-9]{64}\.bin$/ -export const EMBEDDING_CHECKPOINT_CLEANUP_EVENT = 'knowledge.document.embedding-checkpoint.expire' interface CheckpointHeader { version: 1 diff --git a/apps/sim/lib/knowledge/documents/ocr-checkpoints.test.ts b/apps/sim/lib/knowledge/documents/ocr-checkpoints.test.ts index 07caafa7af8..9eddddb08ab 100644 --- a/apps/sim/lib/knowledge/documents/ocr-checkpoints.test.ts +++ b/apps/sim/lib/knowledge/documents/ocr-checkpoints.test.ts @@ -25,12 +25,12 @@ import { PDFDocument } from 'pdf-lib' import { env } from '@/lib/core/config/env' import type { OutboxEventContext } from '@/lib/core/outbox/service' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT } from '@/lib/knowledge/documents/checkpoint-events' import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' import { processDocument } from '@/lib/knowledge/documents/document-processor' import { cleanupOcrCheckpoint, createOcrCheckpoints, - OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, } from '@/lib/knowledge/documents/ocr-checkpoints' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' diff --git a/apps/sim/lib/knowledge/documents/ocr-checkpoints.ts b/apps/sim/lib/knowledge/documents/ocr-checkpoints.ts index d5cc83b03eb..78ca7906804 100644 --- a/apps/sim/lib/knowledge/documents/ocr-checkpoints.ts +++ b/apps/sim/lib/knowledge/documents/ocr-checkpoints.ts @@ -5,6 +5,7 @@ import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' import { deferOutboxHandler, type OutboxHandler } from '@/lib/core/outbox/service' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT } from '@/lib/knowledge/documents/checkpoint-events' import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' import type { OcrRequestPolicy } from '@/lib/knowledge/documents/ocr-request-policy' import { @@ -27,8 +28,6 @@ const MAX_CHECKPOINT_TEXT_BYTES = 20 * 1024 * 1024 const CHECKPOINT_KEY_PATTERN = /^knowledge-ocr-checkpoints\/v1\/[a-f0-9]{64}\/\d{1,5}--?\d{1,5}\.txt$/ -export const OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT = 'knowledge.document.ocr-checkpoint.expire' - /** Canonical internal processing identity, never populated from a public request. */ export interface OcrCheckpointContext { knowledgeBaseId: string diff --git a/apps/sim/lib/knowledge/documents/processing-continuation-dispatch.ts b/apps/sim/lib/knowledge/documents/processing-continuation-dispatch.ts index b516f158969..27115ce2e88 100644 --- a/apps/sim/lib/knowledge/documents/processing-continuation-dispatch.ts +++ b/apps/sim/lib/knowledge/documents/processing-continuation-dispatch.ts @@ -3,6 +3,7 @@ import { outboxEvent } from '@sim/db/schema' import { tasks } from '@trigger.dev/sdk' import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' import { isTriggerAvailable } from '@/lib/core/config/trigger-availability' +import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-event' import { documentProcessingQueueOptions } from '@/lib/knowledge/documents/processing-lane' import type { DocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' @@ -11,8 +12,6 @@ export interface DocumentProcessingContinuation { readonly processingQueueToken: string } -export const KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT = 'knowledge.document.processing.resume' - /** * Uses the deployment's existing durable worker. The outbox path covers ordinary * KB uploads on installations without Trigger.dev, retaining the indexing pass. diff --git a/apps/sim/lib/knowledge/documents/processing-continuation-event.ts b/apps/sim/lib/knowledge/documents/processing-continuation-event.ts new file mode 100644 index 00000000000..dd8d1321016 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-continuation-event.ts @@ -0,0 +1 @@ +export const KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT = 'knowledge.document.processing.resume' diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts index 478c7babca6..57bf710d93d 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts @@ -9,33 +9,27 @@ import { } from '@/lib/core/outbox/service' import { isBYOKEmbeddingCredentialRejection, isEmbeddingQuotaExhaustion } from '@/lib/embeddings' import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' +import { cleanupKnowledgeConnector } from '@/lib/knowledge/connectors/deletion' +import { detachKnowledgeConnector } from '@/lib/knowledge/connectors/detachment' import { - cleanupKnowledgeConnector, KNOWLEDGE_CONNECTOR_CLEANUP_EVENT, -} from '@/lib/knowledge/connectors/deletion' -import { - detachKnowledgeConnector, KNOWLEDGE_CONNECTOR_DETACH_EVENT, -} from '@/lib/knowledge/connectors/detachment' +} from '@/lib/knowledge/connectors/outbox-events' +import { + EMBEDDING_CHECKPOINT_CLEANUP_EVENT, + OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, +} from '@/lib/knowledge/documents/checkpoint-events' import { checkDeferredDocumentRetry } from '@/lib/knowledge/documents/deferred-retry-check' import { getOcrRequestRejection, isPermanentDocumentProcessingError, isUsageLimitDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' -import { - cleanupEmbeddingCheckpoint, - EMBEDDING_CHECKPOINT_CLEANUP_EVENT, -} from '@/lib/knowledge/documents/embedding-checkpoints' -import { - cleanupOcrCheckpoint, - OCR_CHECKPOINT_CLEANUP_OUTBOX_EVENT, -} from '@/lib/knowledge/documents/ocr-checkpoints' +import { cleanupEmbeddingCheckpoint } from '@/lib/knowledge/documents/embedding-checkpoints' +import { cleanupOcrCheckpoint } from '@/lib/knowledge/documents/ocr-checkpoints' import { reclaimStaleDocumentProcessingClaim } from '@/lib/knowledge/documents/processing-claim' -import { - dispatchDocumentProcessingContinuation, - KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT, -} from '@/lib/knowledge/documents/processing-continuation-dispatch' +import { dispatchDocumentProcessingContinuation } from '@/lib/knowledge/documents/processing-continuation-dispatch' +import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-event' import { KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT, KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, diff --git a/apps/sim/lib/mothership/inbox/cleanup-event.ts b/apps/sim/lib/mothership/inbox/cleanup-event.ts new file mode 100644 index 00000000000..c26608ca217 --- /dev/null +++ b/apps/sim/lib/mothership/inbox/cleanup-event.ts @@ -0,0 +1 @@ +export const INBOX_CLEANUP_EVENT = 'inbox.resources.cleanup' diff --git a/apps/sim/lib/mothership/inbox/cleanup-outbox.ts b/apps/sim/lib/mothership/inbox/cleanup-outbox.ts index 0c37df5315c..f5e10a64881 100644 --- a/apps/sim/lib/mothership/inbox/cleanup-outbox.ts +++ b/apps/sim/lib/mothership/inbox/cleanup-outbox.ts @@ -12,9 +12,9 @@ import { processOutboxEventById, } from '@/lib/core/outbox/service' import * as agentmail from '@/lib/mothership/inbox/agentmail-client' +import { INBOX_CLEANUP_EVENT } from '@/lib/mothership/inbox/cleanup-event' const logger = createLogger('InboxCleanup') -const INBOX_CLEANUP_EVENT = 'inbox.resources.cleanup' const MAX_DELETION_POLLS = 120 const DELETION_POLL_INTERVAL_MS = 30_000 diff --git a/apps/sim/lib/organizations/resource-cleanup-event.ts b/apps/sim/lib/organizations/resource-cleanup-event.ts new file mode 100644 index 00000000000..e9621b19fa4 --- /dev/null +++ b/apps/sim/lib/organizations/resource-cleanup-event.ts @@ -0,0 +1 @@ +export const ORGANIZATION_RESOURCE_CLEANUP_EVENT = 'organization.resources.cleanup' diff --git a/apps/sim/lib/organizations/resource-cleanup.test.ts b/apps/sim/lib/organizations/resource-cleanup.test.ts index b139ad80093..f0be632af0e 100644 --- a/apps/sim/lib/organizations/resource-cleanup.test.ts +++ b/apps/sim/lib/organizations/resource-cleanup.test.ts @@ -17,9 +17,9 @@ vi.mock('@/lib/core/outbox/service', () => outboxServiceMock) import type { OutboxEventContext } from '@/lib/core/outbox/service' import { enqueueOrganizationResourceCleanup, - ORGANIZATION_RESOURCE_CLEANUP_EVENT, organizationResourceCleanupOutboxHandlers, } from '@/lib/organizations/resource-cleanup' +import { ORGANIZATION_RESOURCE_CLEANUP_EVENT } from '@/lib/organizations/resource-cleanup-event' const { mockDeleteFile } = storageServiceMockFns const { mockEnqueueOutboxEvent } = outboxServiceMockFns diff --git a/apps/sim/lib/organizations/resource-cleanup.ts b/apps/sim/lib/organizations/resource-cleanup.ts index 36efd53c4af..d6eb271cc10 100644 --- a/apps/sim/lib/organizations/resource-cleanup.ts +++ b/apps/sim/lib/organizations/resource-cleanup.ts @@ -8,9 +8,9 @@ import { env } from '@/lib/core/config/env' import { enqueueOutboxEvent, type OutboxHandlerRegistry } from '@/lib/core/outbox/service' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import type { DbTransaction } from '@/lib/db/types' +import { ORGANIZATION_RESOURCE_CLEANUP_EVENT } from '@/lib/organizations/resource-cleanup-event' import { deleteFile } from '@/lib/uploads/core/storage-service' -export const ORGANIZATION_RESOURCE_CLEANUP_EVENT = 'organization.resources.cleanup' const CLEANUP_BATCH_SIZE = 100 const FILE_DELETE_CONCURRENCY = 10 diff --git a/apps/sim/lib/uploads/contexts/workspace/__integration__/file-versions.integration.ts b/apps/sim/lib/uploads/contexts/workspace/__integration__/file-versions.integration.ts index e88fdc4cab6..21ab15dacce 100644 --- a/apps/sim/lib/uploads/contexts/workspace/__integration__/file-versions.integration.ts +++ b/apps/sim/lib/uploads/contexts/workspace/__integration__/file-versions.integration.ts @@ -31,6 +31,7 @@ import { } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { createFileReadTransport } from '@/lib/mothership/agent-cli/file-read-transport' import { runCli } from '@/lib/mothership/agent-cli/run-cli' +import { WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT } from '@/lib/uploads/contexts/workspace/file-outbox-events' import { deleteWorkspaceFileVersion, fetchWorkspaceFileBuffer, @@ -40,7 +41,6 @@ import { uploadWorkspaceFile, } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT } from '@/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox' import { getCurrentWorkspaceFileVersion, getWorkspaceFileVersion, diff --git a/apps/sim/lib/uploads/contexts/workspace/file-outbox-events.ts b/apps/sim/lib/uploads/contexts/workspace/file-outbox-events.ts new file mode 100644 index 00000000000..fd3b1d8e2a0 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/workspace/file-outbox-events.ts @@ -0,0 +1,2 @@ +export const WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT = 'workspace-file.live-doc.reconcile' +export const WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT = 'workspace-file.storage.cleanup' diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts index 4a2a0cee949..750b238e9e5 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts @@ -8,10 +8,8 @@ vi.mock('@/lib/realtime/notify', () => realtimeNotifyMock) vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) import type { OutboxEventContext } from '@/lib/core/outbox/service' -import { - WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, - workspaceFileLiveDocOutboxHandlers, -} from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' +import { WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT } from '@/lib/uploads/contexts/workspace/file-outbox-events' +import { workspaceFileLiveDocOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' const mockDownloadFile = storageServiceMockFns.mockDownloadFile diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts index 5043a58f974..a4f06884946 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts @@ -11,11 +11,10 @@ import { processOutboxEventById, } from '@/lib/core/outbox/service' import { applyEditToLiveFileDoc, invalidateLiveFileDoc } from '@/lib/realtime/notify' +import { WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT } from '@/lib/uploads/contexts/workspace/file-outbox-events' import { downloadFile } from '@/lib/uploads/core/storage-service' import { isMarkdownFile } from '@/lib/uploads/utils/file-utils' -export const WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT = 'workspace-file.live-doc.reconcile' - interface WorkspaceFileLiveDocPayload { workspaceId: string fileId: string diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox.test.ts index 02d1f1edb35..d8b9a2bd073 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox.test.ts @@ -15,10 +15,10 @@ vi.mock('@/lib/core/outbox/service', () => ({ vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) import type { OutboxEventContext } from '@/lib/core/outbox/service' +import { WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT } from '@/lib/uploads/contexts/workspace/file-outbox-events' import { enqueueWorkspaceFileStorageCleanups, processWorkspaceFileStorageCleanupsNow, - WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT, workspaceFileStorageCleanupOutboxHandlers, } from '@/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox' diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox.ts index e97dd72278e..934df6450fd 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox.ts @@ -10,12 +10,11 @@ import { type OutboxHandlerRegistry, processOutboxEventById, } from '@/lib/core/outbox/service' +import { WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT } from '@/lib/uploads/contexts/workspace/file-outbox-events' import { deleteFile } from '@/lib/uploads/core/storage-service' const logger = createLogger('WorkspaceFileStorageCleanup') -export const WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT = 'workspace-file.storage.cleanup' - interface WorkspaceFileStorageCleanupPayload { key: string } diff --git a/apps/sim/lib/workflows/application/mapped-import.ts b/apps/sim/lib/workflows/application/mapped-import.ts index 7942a9130da..7832fcd72f3 100644 --- a/apps/sim/lib/workflows/application/mapped-import.ts +++ b/apps/sim/lib/workflows/application/mapped-import.ts @@ -48,6 +48,7 @@ import { type ActiveWorkspaceApplicationContext, resolveActiveWorkspaceApplicationContext, } from '@/lib/workspaces/application/workspace-context' +import { WORKSPACE_WORKFLOWS_CHANGED_EVENT } from '@/lib/workspaces/operations/outbox-events' import { findWorkspaceOperationReceipt, insertWorkspaceOperationReceipt, @@ -342,7 +343,7 @@ export async function applyMappedWorkflowImport( }, } await insertWorkspaceOperationReceipt(tx, requestHash, report) - await enqueueOutboxEvent(tx, 'workspace.workflows.changed', { + await enqueueOutboxEvent(tx, WORKSPACE_WORKFLOWS_CHANGED_EVENT, { workspaceId: context.workspaceId, }) return receiptResult(report, false) diff --git a/apps/sim/lib/workflows/deployment-outbox-events.ts b/apps/sim/lib/workflows/deployment-outbox-events.ts new file mode 100644 index 00000000000..548ddee8833 --- /dev/null +++ b/apps/sim/lib/workflows/deployment-outbox-events.ts @@ -0,0 +1,8 @@ +export const WORKFLOW_DEPLOYMENT_OUTBOX_EVENTS = { + PREPARE_V2: 'workflow.deployment.prepare.v2', + /** One-release rolling compatibility for events admitted by pre-v2 pods. */ + SYNC_ACTIVE_SIDE_EFFECTS: 'workflow.deployment.sync-active-side-effects', + /** One-release rolling compatibility for cleanup admitted by pre-v2 pods. */ + CLEANUP_INACTIVE_SIDE_EFFECTS: 'workflow.deployment.cleanup-inactive-side-effects', + CLEANUP_UNDEPLOYED_SIDE_EFFECTS: 'workflow.deployment.cleanup-undeployed-side-effects', +} as const diff --git a/apps/sim/lib/workflows/deployment-outbox.test.ts b/apps/sim/lib/workflows/deployment-outbox.test.ts index 809723eb7e5..9f3c09ca3de 100644 --- a/apps/sim/lib/workflows/deployment-outbox.test.ts +++ b/apps/sim/lib/workflows/deployment-outbox.test.ts @@ -115,8 +115,8 @@ import { NonRetryableDeploymentError } from '@/lib/workflows/deployment-lifecycl import { createWorkflowDeploymentOutboxHandlers, type PrepareDeploymentV2Payload, - WORKFLOW_DEPLOYMENT_OUTBOX_EVENTS, } from '@/lib/workflows/deployment-outbox' +import { WORKFLOW_DEPLOYMENT_OUTBOX_EVENTS } from '@/lib/workflows/deployment-outbox-events' const mockTx = dbChainMock.db diff --git a/apps/sim/lib/workflows/deployment-outbox.ts b/apps/sim/lib/workflows/deployment-outbox.ts index 26d157e583f..566fc643351 100644 --- a/apps/sim/lib/workflows/deployment-outbox.ts +++ b/apps/sim/lib/workflows/deployment-outbox.ts @@ -47,6 +47,7 @@ import { NonRetryableDeploymentError, parseDeploymentReadiness, } from '@/lib/workflows/deployment-lifecycle' +import { WORKFLOW_DEPLOYMENT_OUTBOX_EVENTS } from '@/lib/workflows/deployment-outbox-events' import { activateDeploymentOperation, beginDeploymentOperationActivation, @@ -74,15 +75,6 @@ import type { BlockState } from '@/stores/workflows/workflow/types' const logger = createLogger('WorkflowDeploymentOutbox') -export const WORKFLOW_DEPLOYMENT_OUTBOX_EVENTS = { - PREPARE_V2: 'workflow.deployment.prepare.v2', - /** One-release rolling compatibility for events admitted by pre-v2 pods. */ - SYNC_ACTIVE_SIDE_EFFECTS: 'workflow.deployment.sync-active-side-effects', - /** One-release rolling compatibility for cleanup admitted by pre-v2 pods. */ - CLEANUP_INACTIVE_SIDE_EFFECTS: 'workflow.deployment.cleanup-inactive-side-effects', - CLEANUP_UNDEPLOYED_SIDE_EFFECTS: 'workflow.deployment.cleanup-undeployed-side-effects', -} as const - export const DEPLOYMENT_READINESS_COMPONENTS = ['webhooks', 'schedules', 'mcp'] as const /** diff --git a/apps/sim/lib/workspaces/admin-move-event.ts b/apps/sim/lib/workspaces/admin-move-event.ts new file mode 100644 index 00000000000..e9b36bfbc18 --- /dev/null +++ b/apps/sim/lib/workspaces/admin-move-event.ts @@ -0,0 +1 @@ +export const MIGRATED_INVITATION_EMAIL_EVENT_TYPE = 'invitation.send-migrated-link' diff --git a/apps/sim/lib/workspaces/admin-move.test.ts b/apps/sim/lib/workspaces/admin-move.test.ts index 958893aabb6..5336e79af5c 100644 --- a/apps/sim/lib/workspaces/admin-move.test.ts +++ b/apps/sim/lib/workspaces/admin-move.test.ts @@ -35,10 +35,10 @@ import { getWorkspaceMoveOperation, getWorkspaceMovePreflight, invitationMigrationOutboxHandlers, - MIGRATED_INVITATION_EMAIL_EVENT_TYPE, moveWorkspaceToOrganization, projectDestinationPendingSeatCount, } from '@/lib/workspaces/admin-move' +import { MIGRATED_INVITATION_EMAIL_EVENT_TYPE } from '@/lib/workspaces/admin-move-event' import { WORKSPACE_MODE } from '@/lib/workspaces/policy' vi.unmock('drizzle-orm') diff --git a/apps/sim/lib/workspaces/admin-move.ts b/apps/sim/lib/workspaces/admin-move.ts index f0a98b5773d..6c3f9f6f86a 100644 --- a/apps/sim/lib/workspaces/admin-move.ts +++ b/apps/sim/lib/workspaces/admin-move.ts @@ -44,6 +44,7 @@ import { PENDING_INVITATION_UNIQUE_INDEX, sendInvitationEmail } from '@/lib/invi import { ProjectConflictError, transferWorkspaceProjects } from '@/lib/projects/membership' import { invalidateWorkspaceTableLimitsCache } from '@/lib/table/billing' import { deleteCustomBlock } from '@/lib/workflows/custom-blocks/operations' +import { MIGRATED_INVITATION_EMAIL_EVENT_TYPE } from '@/lib/workspaces/admin-move-event' import { type CrossOrgForkEdge, cleanupSourceOrganizationArtifactsTx, @@ -316,7 +317,6 @@ interface MoveTransactionResult { summary: WorkspaceMovePreflight } -export const MIGRATED_INVITATION_EMAIL_EVENT_TYPE = 'invitation.send-migrated-link' export const ADMIN_WORKSPACE_MOVE_OPERATION_EVENT_TYPE = 'admin.workspace-move-operation' interface AdminWorkspaceMoveOperationRequest { diff --git a/apps/sim/lib/workspaces/operations/outbox-events.ts b/apps/sim/lib/workspaces/operations/outbox-events.ts new file mode 100644 index 00000000000..cad08952ee0 --- /dev/null +++ b/apps/sim/lib/workspaces/operations/outbox-events.ts @@ -0,0 +1,3 @@ +export const WORKSPACE_MCP_CHANGED_EVENT = 'workspace.mcp.changed' +export const WORKSPACE_OPERATION_OBSERVE_EVENT = 'workspace.operation.observe' +export const WORKSPACE_WORKFLOWS_CHANGED_EVENT = 'workspace.workflows.changed' diff --git a/apps/sim/lib/workspaces/operations/outbox.ts b/apps/sim/lib/workspaces/operations/outbox.ts index 3a735359819..85a035a98b0 100644 --- a/apps/sim/lib/workspaces/operations/outbox.ts +++ b/apps/sim/lib/workspaces/operations/outbox.ts @@ -2,6 +2,11 @@ import { z } from 'zod' import { deferOutboxHandler, type OutboxHandlerRegistry } from '@/lib/core/outbox/service' import { publishMcpToolServerChanges } from '@/lib/mcp/workflow-mcp-sync' import { notifyWorkspaceWorkflowsChanged } from '@/lib/realtime/notify' +import { + WORKSPACE_MCP_CHANGED_EVENT, + WORKSPACE_OPERATION_OBSERVE_EVENT, + WORKSPACE_WORKFLOWS_CHANGED_EVENT, +} from '@/lib/workspaces/operations/outbox-events' import { refreshWorkspaceOperation } from '@/lib/workspaces/operations/refresh' const operationSchema = z @@ -14,16 +19,16 @@ const serverChangesSchema = z .strict() export const workspaceOperationOutboxHandlers = { - 'workspace.mcp.changed': async (payload) => { + [WORKSPACE_MCP_CHANGED_EVENT]: async (payload) => { await publishMcpToolServerChanges(serverChangesSchema.parse(payload).serverIds) }, - 'workspace.operation.observe': async (payload) => { + [WORKSPACE_OPERATION_OBSERVE_EVENT]: async (payload) => { const { workspaceId, operationId } = operationSchema.parse(payload) const report = await refreshWorkspaceOperation(workspaceId, operationId) if (report && !report.completionRecorded) return deferOutboxHandler('Waiting for workspace operation effects', 5000, false) }, - 'workspace.workflows.changed': async (payload) => { + [WORKSPACE_WORKFLOWS_CHANGED_EVENT]: async (payload) => { const { workspaceId } = changedWorkspaceSchema.parse(payload) await notifyWorkspaceWorkflowsChanged(workspaceId) }, diff --git a/packages/testing/src/mocks/billing-outbox-handlers.mock.ts b/packages/testing/src/mocks/billing-outbox-handlers.mock.ts index dba3baa5980..ef48171a2ff 100644 --- a/packages/testing/src/mocks/billing-outbox-handlers.mock.ts +++ b/packages/testing/src/mocks/billing-outbox-handlers.mock.ts @@ -1,7 +1,7 @@ import { vi } from 'vitest' /** - * Real `OUTBOX_EVENT_TYPES` values from `@/lib/billing/webhooks/outbox-handlers`. + * Real `OUTBOX_EVENT_TYPES` values from `@/lib/billing/webhooks/outbox-events`. */ const OUTBOX_EVENT_TYPES = { STRIPE_SYNC_CANCEL_AT_PERIOD_END: 'stripe.sync-cancel-at-period-end', @@ -32,8 +32,8 @@ export const billingOutboxHandlersMockFns = { } /** - * Static mock module for `@/lib/billing/webhooks/outbox-handlers`. `OUTBOX_EVENT_TYPES` carries - * the real values; `billingOutboxHandlers` maps every event type to its mock handler. + * Static mock module for `@/lib/billing/webhooks/outbox-handlers`: `billingOutboxHandlers` maps + * every real event type to its mock handler. * * @example * ```ts @@ -41,7 +41,6 @@ export const billingOutboxHandlersMockFns = { * ``` */ export const billingOutboxHandlersMock = { - OUTBOX_EVENT_TYPES, billingOutboxHandlers: { [OUTBOX_EVENT_TYPES.STRIPE_SYNC_CANCEL_AT_PERIOD_END]: billingOutboxHandlersMockFns.mockStripeSyncCancelAtPeriodEnd, diff --git a/packages/testing/src/mocks/organization-membership.mock.ts b/packages/testing/src/mocks/organization-membership.mock.ts index df9fb895862..4495d9fe1fc 100644 --- a/packages/testing/src/mocks/organization-membership.mock.ts +++ b/packages/testing/src/mocks/organization-membership.mock.ts @@ -41,9 +41,8 @@ export const organizationMembershipMockFns = { } /** - * Static mock module for `@/lib/billing/organizations/membership`, including its constants - * (`MEMBER_BILLING_RECONCILIATION_EVENT_TYPE` and the re-exported - * `WORKSPACE_BILLING_ACCOUNT_REMOVAL_ERROR`) with their real values. + * Static mock module for `@/lib/billing/organizations/membership`, including its re-exported + * `WORKSPACE_BILLING_ACCOUNT_REMOVAL_ERROR` constant with its real value. * * @example * ```ts @@ -51,7 +50,6 @@ export const organizationMembershipMockFns = { * ``` */ export const organizationMembershipMock = { - MEMBER_BILLING_RECONCILIATION_EVENT_TYPE: 'billing.reconcile-member-after-org-leave', WORKSPACE_BILLING_ACCOUNT_REMOVAL_ERROR: 'Cannot remove the workspace billing account. Please reassign billing first.', acquireUserBillingIdentityLock: organizationMembershipMockFns.mockAcquireUserBillingIdentityLock, diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index 84e2a8a407d..6d11aa2dbc4 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -1252,7 +1252,6 @@ "apps/sim/hooks/use-user-permissions.ts#PermissionType", "apps/sim/lib/a2a/client.ts#A2AArtifactOutput", "apps/sim/lib/a2a/client.ts#extractText", - "apps/sim/lib/admin/member-operation.ts#ADMIN_MEMBER_OPERATION_EVENT_TYPE", "apps/sim/lib/api-key/application/operations.ts#ApiKeyOperation", "apps/sim/lib/api-key/application/operations.ts#ByokKeyOperation", "apps/sim/lib/api-key/application/organization-byok-keys.ts#OrganizationByokKeyMutationProjection", @@ -4199,8 +4198,6 @@ "apps/sim/lib/billing/enterprise-outbox.ts#enterpriseProvisionRequestSchema", "apps/sim/lib/billing/enterprise-outbox.ts#getLatestEnterpriseIssuanceForOrganization", "apps/sim/lib/billing/enterprise-owner-claim.ts#AcceptEnterpriseOwnerClaimFailure", - "apps/sim/lib/billing/enterprise-owner-claim.ts#ENTERPRISE_OWNER_ACTIVATION_EVENT_TYPE", - "apps/sim/lib/billing/enterprise-owner-claim.ts#ENTERPRISE_OWNER_CLAIM_EVENT_TYPE", "apps/sim/lib/billing/enterprise-owner-claim.ts#ENTERPRISE_OWNER_CLAIM_EXPIRY_DAYS", "apps/sim/lib/billing/enterprise-owner-claim.ts#enterpriseOwnerClaimPayloadSchema", "apps/sim/lib/billing/enterprise-owner-claim.ts#getEnterpriseOwnerClaimView", From 0003dfc68fef63c5184c02b4216fb7e72e81403d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 23:57:45 -0700 Subject: [PATCH 2/5] refactor(outbox): take only lazy handler groups; resolve eligible types in one place --- apps/sim/lib/core/outbox/handlers.ts | 7 +-- .../lib/core/outbox/service.integration.ts | 59 +++++++++++------- apps/sim/lib/core/outbox/service.test.ts | 62 +++++++++---------- apps/sim/lib/core/outbox/service.ts | 56 +++++++---------- 4 files changed, 91 insertions(+), 93 deletions(-) diff --git a/apps/sim/lib/core/outbox/handlers.ts b/apps/sim/lib/core/outbox/handlers.ts index 3860af6500b..b891c745843 100644 --- a/apps/sim/lib/core/outbox/handlers.ts +++ b/apps/sim/lib/core/outbox/handlers.ts @@ -57,10 +57,9 @@ import { FORK_CONTENT_COPY_EVENT } from '@/ee/workspace-forking/application/cont * the dependencies of handlers it will not call. Event types come from dependency-free modules * (or ones the processor already loads), so naming them here costs nothing. * - * Each group's `events` must equal its loaded map's keys: a type missing here is never served and - * dead-letters as unhandled (`handlers.test.ts` enforces this). Each `load` destructures its - * import so the unused-export audit can see which export it reads; `(await import(x)).y` marks - * every export of the module as used. + * `handlers.test.ts` checks every group against its module. Each `load` destructures its import + * so the unused-export audit can see which export it reads; `(await import(x)).y` marks every + * export of the module as used. */ export const OUTBOX_HANDLER_GROUPS: readonly LazyOutboxHandlerGroup[] = [ { diff --git a/apps/sim/lib/core/outbox/service.integration.ts b/apps/sim/lib/core/outbox/service.integration.ts index 589c41c5659..d1b77efc066 100644 --- a/apps/sim/lib/core/outbox/service.integration.ts +++ b/apps/sim/lib/core/outbox/service.integration.ts @@ -26,11 +26,18 @@ import { } from '@/lib/core/outbox/queries' import { hasDueOutboxWork, + type LazyOutboxHandlerGroup, type OutboxHandler, + type OutboxHandlerRegistry, processOutboxEvents, withOutboxHandlerTimeout, } from '@/lib/core/outbox/service' +/** Serves an eager handler map as one group, the shape `processOutboxEvents` takes. */ +const asGroups = (handlers: OutboxHandlerRegistry): LazyOutboxHandlerGroup[] => [ + { events: Object.keys(handlers), load: async () => handlers }, +] + interface QueryPlan { 'Node Type': string 'Index Name'?: string @@ -196,7 +203,9 @@ describe('outbox scheduling in PostgreSQL', () => { { eventType: 'test.outbox.z-ready' }, ]) await expectBoundedDiscovery(now) - expect(await processOutboxEvents({ 'test.outbox.z-ready': async () => {} })).toMatchObject({ + expect( + await processOutboxEvents(asGroups({ 'test.outbox.z-ready': async () => {} })) + ).toMatchObject({ processed: 1, }) expect(await db.execute(readyEventTypesQuery(new Date()))).toEqual([]) @@ -205,7 +214,7 @@ describe('outbox scheduling in PostgreSQL', () => { it('retains bounded retries for a type missing during a rolling deployment', async () => { const [event] = await enqueue('test.outbox.unknown', 1) - expect(await processOutboxEvents({})).toMatchObject({ retried: 1 }) + expect(await processOutboxEvents(asGroups({}))).toMatchObject({ retried: 1 }) const [pending] = await db.select().from(outboxEvent).where(eq(outboxEvent.id, event.id)) expect(pending).toMatchObject({ status: 'pending', attempts: 1 }) expect(pending.availableAt.getTime()).toBeGreaterThan(Date.now()) @@ -216,11 +225,13 @@ describe('outbox scheduling in PostgreSQL', () => { const delivered: string[] = [] await connection.begin(async (transaction) => { await transaction`SELECT id FROM outbox_event WHERE id = ${locked.id} FOR UPDATE` - const result = await processOutboxEvents({ - 'test.outbox.locked': async (_payload, context) => { - delivered.push(context.eventId) - }, - }) + const result = await processOutboxEvents( + asGroups({ + 'test.outbox.locked': async (_payload, context) => { + delivered.push(context.eventId) + }, + }) + ) expect(result.processed).toBe(1) }) expect(delivered).toEqual([available.id]) @@ -236,11 +247,11 @@ describe('outbox scheduling in PostgreSQL', () => { } const result = await processOutboxEvents( - { + asGroups({ 'test.outbox.cleanup': handler, 'test.outbox.dispatch': handler, 'test.outbox.billing': handler, - }, + }), { batchSize: 20 } ) @@ -256,7 +267,9 @@ describe('outbox scheduling in PostgreSQL', () => { delivered.push(context.eventId) } - const result = await processOutboxEvents({ 'test.outbox.cleanup': handler }, { batchSize: 25 }) + const result = await processOutboxEvents(asGroups({ 'test.outbox.cleanup': handler }), { + batchSize: 25, + }) expect(result.processed).toBe(25) expect(delivered).toEqual(rows.slice(0, 25).map((row) => row.id)) @@ -274,10 +287,12 @@ describe('outbox scheduling in PostgreSQL', () => { delivered.push(context.eventId) } - const result = await processOutboxEvents({ - 'test.outbox.future': handler, - 'test.outbox.cleanup': handler, - }) + const result = await processOutboxEvents( + asGroups({ + 'test.outbox.future': handler, + 'test.outbox.cleanup': handler, + }) + ) expect(result.processed).toBe(5) expect(delivered).not.toContain(future.id) @@ -293,8 +308,8 @@ describe('outbox scheduling in PostgreSQL', () => { const handlers = { 'test.outbox.cleanup': handler, 'test.outbox.dispatch': handler } const results = await Promise.all([ - processOutboxEvents(handlers, { batchSize: 40 }), - processOutboxEvents(handlers, { batchSize: 40 }), + processOutboxEvents(asGroups(handlers), { batchSize: 40 }), + processOutboxEvents(asGroups(handlers), { batchSize: 40 }), ]) expect(results.reduce((sum, result) => sum + result.processed, 0)).toBe(52) @@ -335,11 +350,11 @@ describe('outbox scheduling in PostgreSQL', () => { delivered.push(context.eventId) } const result = await processOutboxEvents( - { + asGroups({ 'test.outbox.cleanup': handler, 'test.outbox.dispatch': handler, 'test.outbox.billing': handler, - }, + }), { batchSize: 20 } ) @@ -356,12 +371,12 @@ describe('outbox scheduling in PostgreSQL', () => { delivered.push(context.eventId) } const result = await processOutboxEvents( - { + asGroups({ 'test.outbox.long': withOutboxHandlerTimeout(async () => { throw new Error('Long handler must remain pending') }, 550_000), 'test.outbox.short': handler, - }, + }), { maxRuntimeMs: 110_000 } ) @@ -374,7 +389,7 @@ describe('outbox scheduling in PostgreSQL', () => { it('bounds stale-lease recovery and leaves excess rows for the next invocation', async () => { await seedBacklog('test.outbox.stale', 1_005, 'processing') - const first = await processOutboxEvents({}, { batchSize: 0 }) + const first = await processOutboxEvents(asGroups({}), { batchSize: 0 }) expect(first.reaped).toBe(1_000) const counts = await db .select({ status: outboxEvent.status, count: sql`count(*)::int` }) @@ -388,7 +403,7 @@ describe('outbox scheduling in PostgreSQL', () => { ]) ) - const second = await processOutboxEvents({}, { batchSize: 0 }) + const second = await processOutboxEvents(asGroups({}), { batchSize: 0 }) expect(second.reaped).toBe(5) }) diff --git a/apps/sim/lib/core/outbox/service.test.ts b/apps/sim/lib/core/outbox/service.test.ts index 692a93f482f..356e1693972 100644 --- a/apps/sim/lib/core/outbox/service.test.ts +++ b/apps/sim/lib/core/outbox/service.test.ts @@ -26,6 +26,8 @@ import { deferOutboxHandler, enqueueOrReschedulePendingOutboxEvent, enqueueOutboxEvents, + type LazyOutboxHandlerGroup, + type OutboxHandlerRegistry, outboxEventHasSourceOperationId, outboxPayloadHasSourceOperationId, processOutboxEvents, @@ -34,6 +36,11 @@ import { idMockFns.mockGenerateId.mockReturnValue('test-event-id') +/** Serves an eager handler map as one group, the shape `processOutboxEvents` takes. */ +const asGroups = (handlers: OutboxHandlerRegistry): LazyOutboxHandlerGroup[] => [ + { events: Object.keys(handlers), load: async () => handlers }, +] + const logger = vi.mocked(createLogger).mock.results[ vi.mocked(createLogger).mock.calls.findIndex(([name]) => name === 'OutboxService') @@ -195,7 +202,7 @@ describe('processOutboxEvents — empty / no handler', () => { queuePendingEvents([makePendingRow({ eventType: 'unknown.event' })]) holdLease() - const result = await processOutboxEvents({}) + const result = await processOutboxEvents(asGroups({})) expect(result.retried).toBe(1) const retry = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) @@ -209,7 +216,7 @@ describe('processOutboxEvents — empty / no handler', () => { ]) holdLease() - const result = await processOutboxEvents({}) + const result = await processOutboxEvents(asGroups({})) expect(result.deadLettered).toBe(1) const terminal = updateSets().find((set) => set.status === 'dead_letter') @@ -229,7 +236,7 @@ describe('processOutboxEvents — infrastructure diagnostics', () => { const error = new Error('Failed query: select event_type\nparams: private-token', { cause }) dbChainMockFns.execute.mockRejectedValueOnce(error) - await expect(processOutboxEvents({})).rejects.toBe(error) + await expect(processOutboxEvents(asGroups({}))).rejects.toBe(error) expect(logger.error).toHaveBeenCalledWith( 'Outbox processing failed', @@ -255,7 +262,7 @@ describe('processOutboxEvents — infrastructure diagnostics', () => { .mockImplementationOnce(async (callback) => callback(dbChainMock.db)) .mockRejectedValueOnce(error) - await expect(processOutboxEvents({ 'test.event': handler })).rejects.toBe(error) + await expect(processOutboxEvents(asGroups({ 'test.event': handler }))).rejects.toBe(error) expect(logger.error).toHaveBeenCalledWith( 'Outbox processing failed', @@ -279,7 +286,7 @@ describe('processOutboxEvents — handler success and retry', () => { queuePendingEvents([makePendingRow()]) holdLease() - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.processed).toBe(1) expect(handlerCalls).toEqual([{ payload: { foo: 'bar' }, eventId: 'evt-1', attempts: 0 }]) @@ -299,7 +306,7 @@ describe('processOutboxEvents — handler success and retry', () => { ) queuePendingEvents([makePendingRow()]) - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.leaseLost).toBe(1) expect(result.processed).toBe(0) @@ -314,7 +321,7 @@ describe('processOutboxEvents — handler success and retry', () => { holdLease() const before = Date.now() - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.retried).toBe(1) const retryUpdate = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) @@ -332,7 +339,7 @@ describe('processOutboxEvents — handler success and retry', () => { queuePendingEvents([makePendingRow({ attempts: 2 })]) holdLease() - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.retried).toBe(1) const deferredUpdate = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) @@ -344,7 +351,7 @@ describe('processOutboxEvents — handler success and retry', () => { queuePendingEvents([makePendingRow({ attempts: 9, maxAttempts: 10 })]) holdLease() - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.deadLettered).toBe(1) const deadUpdate = updateSets().find((set) => set.status === 'dead_letter') @@ -359,7 +366,7 @@ describe('processOutboxEvents — handler success and retry', () => { queuePendingEvents([makePendingRow({ attempts: 4, maxAttempts: 5 })]) holdLease() - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.retried).toBe(1) const deferredUpdate = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) @@ -371,7 +378,7 @@ describe('processOutboxEvents — handler success and retry', () => { queuePendingEvents([makePendingRow({ attempts: 4, maxAttempts: 5 })]) holdLease() - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.retried).toBe(1) const continuedUpdate = updateSets().find( @@ -388,7 +395,7 @@ describe('processOutboxEvents — handler success and retry', () => { queuePendingEvents([makePendingRow({ attempts: 9, maxAttempts: 10 })]) holdLease() - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.deadLettered).toBe(1) const deadUpdate = updateSets().find((set) => set.status === 'dead_letter') @@ -406,7 +413,7 @@ describe('processOutboxEvents — handler success and retry', () => { holdLease() const before = Date.now() - await processOutboxEvents({ 'test.event': handler }) + await processOutboxEvents(asGroups({ 'test.event': handler })) const retryUpdate = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) expect(retryUpdate).toBeDefined() @@ -429,7 +436,7 @@ describe('processOutboxEvents — lease CAS / reaper race', () => { queuePendingEvents([makePendingRow()]) - const result = await processOutboxEvents({ 'test.event': handler }) + const result = await processOutboxEvents(asGroups({ 'test.event': handler })) expect(result.leaseLost).toBe(1) expect(result.processed).toBe(0) @@ -455,7 +462,9 @@ describe('processOutboxEvents — handler timeout', () => { }, 550_000) queuePendingEvents([makePendingRow()]) holdLease() - const promise = processOutboxEvents({ 'test.event': handler }, { maxRuntimeMs: 790_000 }) + const promise = processOutboxEvents(asGroups({ 'test.event': handler }), { + maxRuntimeMs: 790_000, + }) await vi.advanceTimersByTimeAsync(120_001) expect(await promise).toMatchObject({ processed: 1, leaseLost: 0 }) expect(observedDeadline).toBe(startedAt + 550_000) @@ -469,7 +478,7 @@ describe('processOutboxEvents — handler timeout', () => { queuePendingEvents([makePendingRow()]) holdLease() expect( - await processOutboxEvents({ 'test.event': handler }, { maxRuntimeMs: 110_000 }) + await processOutboxEvents(asGroups({ 'test.event': handler }), { maxRuntimeMs: 110_000 }) ).toMatchObject({ processed: 0, retried: 0 }) expect(handler).not.toHaveBeenCalled() expect(updateSets().some((set) => set.status === 'processing')).toBe(false) @@ -490,7 +499,7 @@ describe('processOutboxEvents — handler timeout', () => { holdLease() const result = await processOutboxEvents( - { 'test.long': longHandler, 'test.short': shortHandler }, + asGroups({ 'test.long': longHandler, 'test.short': shortHandler }), { maxRuntimeMs: 110_000 } ) @@ -509,7 +518,7 @@ describe('processOutboxEvents — handler timeout', () => { queuePendingEvents([makePendingRow({ attempts: 0 })]) holdLease() - const promise = processOutboxEvents({ 'test.event': neverResolves }) + const promise = processOutboxEvents(asGroups({ 'test.event': neverResolves })) // Must exceed DEFAULT_HANDLER_TIMEOUT_MS (90s). await vi.advanceTimersByTimeAsync(90 * 1000 + 1) const result = await promise @@ -539,7 +548,7 @@ describe('processOutboxEvents — handler timeout', () => { queuePendingEvents([makePendingRow({ attempts: 0 })]) holdLease() - const promise = processOutboxEvents({ 'test.event': handler }) + const promise = processOutboxEvents(asGroups({ 'test.event': handler })) await vi.advanceTimersByTimeAsync(90 * 1000 + 1) const result = await promise @@ -560,7 +569,7 @@ describe('processOutboxEvents — reaper recovery', () => { { id: 'stuck-3' }, ]) - const result = await processOutboxEvents({}) + const result = await processOutboxEvents(asGroups({})) expect(result.reaped).toBe(3) expect(result.processed).toBe(0) @@ -658,17 +667,4 @@ describe('processOutboxEvents — lazy handler groups', () => { expect(handler).not.toHaveBeenCalled() expect(updateSets().some((set) => set.status === 'processing')).toBe(false) }) - - it('still retries an event type that no handler module declares', async () => { - const load = vi.fn(async () => ({ 'test.event': vi.fn() })) - queuePendingEvents([makePendingRow({ eventType: 'unknown.event' })]) - holdLease() - - const result = await processOutboxEvents([{ events: ['test.event'], load }]) - - expect(result.retried).toBe(1) - const retry = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) - expect(retry?.attempts).toBe(1) - expect(load).not.toHaveBeenCalled() - }) }) diff --git a/apps/sim/lib/core/outbox/service.ts b/apps/sim/lib/core/outbox/service.ts index 0591d1cabe3..adcd2a22c32 100644 --- a/apps/sim/lib/core/outbox/service.ts +++ b/apps/sim/lib/core/outbox/service.ts @@ -138,29 +138,19 @@ export function withOutboxHandlerTimeout( return Object.assign(handler, { timeoutMs }) } -/** - * Map of `eventType` → handler. Register all handlers in one place - * and pass them to `processOutboxEvents`. - */ +/** Map of `eventType` → handler, served lazily through a {@link LazyOutboxHandlerGroup}. */ export type OutboxHandlerRegistry = Record /** * A handler module imported only when one of its event types is due. `events` lists exactly - * the keys of the registry `load` resolves to; a type absent from `events` is never served. + * the keys of the registry `load` resolves to; a type absent from `events` is never served and + * dead-letters as unhandled. */ export interface LazyOutboxHandlerGroup { readonly events: readonly string[] readonly load: () => Promise } -type OutboxHandlerSource = OutboxHandlerRegistry | readonly LazyOutboxHandlerGroup[] - -function isLazyOutboxHandlerSource( - source: OutboxHandlerSource -): source is readonly LazyOutboxHandlerGroup[] { - return Array.isArray(source) -} - export interface EnqueueOptions { /** Caller-owned idempotency key. Defaults to a generated UUID. */ id?: string @@ -445,11 +435,11 @@ export async function hasInflightOutboxEvent( * available events first. Safe to call concurrently from multiple workers — * `SELECT FOR UPDATE SKIP LOCKED` serializes claims. * - * Lazy handler groups are imported for the ready event types before any claim, - * so import time never runs inside a handler's window or a claimed lease. + * Handler groups are imported for the ready event types before any claim, so + * import time never runs inside a handler's window or a claimed lease. */ export async function processOutboxEvents( - handlerSource: OutboxHandlerSource, + handlerGroups: readonly LazyOutboxHandlerGroup[], options: { batchSize?: number; maxRuntimeMs?: number; minRemainingMs?: number } = {} ): Promise { const startedAt = Date.now() @@ -467,13 +457,9 @@ export async function processOutboxEvents( reaped = await reapStuckProcessingRows() phase = 'discover' const readyTypes = await db.execute<{ eventType: string }>(readyEventTypesQuery(new Date())) - const readyEventTypes = readyTypes.map(({ eventType }) => eventType) - const { handlers, unavailableEventTypes } = await resolveOutboxHandlers( - handlerSource, - readyEventTypes - ) - const eligibleTypes = readyEventTypes.filter( - (eventType) => !unavailableEventTypes.has(eventType) + const { handlers, eligibleTypes } = await resolveOutboxHandlers( + handlerGroups, + readyTypes.map(({ eventType }) => eventType) ) let cursor = 0 let claimed = 0 @@ -528,20 +514,19 @@ export async function processOutboxEvents( } /** - * Imports the lazy groups that serve any ready event type. A group whose import fails leaves its - * event types unclaimed for this run: unlike a missing handler, which spends an attempt and - * eventually dead-letters, a failed import says nothing about the events, so they stay pending - * for a later run. Event types outside every group still reach the missing-handler path. + * Imports the groups that serve any ready event type and returns the types to claim this run. A + * group whose import fails leaves its event types unclaimed: unlike a missing handler, which + * spends an attempt and eventually dead-letters, a failed import says nothing about the events, + * so they stay pending for a later run. Event types outside every group stay eligible and reach + * the missing-handler path. */ async function resolveOutboxHandlers( - source: OutboxHandlerSource, - readyEventTypes: readonly string[] -): Promise<{ handlers: OutboxHandlerRegistry; unavailableEventTypes: Set }> { + groups: readonly LazyOutboxHandlerGroup[], + readyEventTypes: string[] +): Promise<{ handlers: OutboxHandlerRegistry; eligibleTypes: string[] }> { const unavailableEventTypes = new Set() - if (!isLazyOutboxHandlerSource(source)) return { handlers: source, unavailableEventTypes } - const ready = new Set(readyEventTypes) - const dueGroups = source.filter((group) => group.events.some((eventType) => ready.has(eventType))) + const dueGroups = groups.filter((group) => group.events.some((eventType) => ready.has(eventType))) const loaded = await Promise.allSettled(dueGroups.map((group) => group.load())) const handlers: OutboxHandlerRegistry = {} for (const [index, outcome] of loaded.entries()) { @@ -559,7 +544,10 @@ async function resolveOutboxHandlers( if (handler) handlers[eventType] = handler } } - return { handlers, unavailableEventTypes } + return { + handlers, + eligibleTypes: readyEventTypes.filter((eventType) => !unavailableEventTypes.has(eventType)), + } } /** From 1badbada4cab310b1c7f04821728685ea5d1eaf3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 00:09:05 -0700 Subject: [PATCH 3/5] fix(outbox): fail the run after maintenance when a handler module could not load --- apps/sim/lib/core/outbox/enqueue.test.ts | 18 +++++++++++++++-- apps/sim/lib/core/outbox/processor.test.ts | 23 +++++++++++++++++++++- apps/sim/lib/core/outbox/processor.ts | 6 ++++++ apps/sim/lib/core/outbox/service.test.ts | 6 +++++- apps/sim/lib/core/outbox/service.ts | 17 +++++++++++----- 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/apps/sim/lib/core/outbox/enqueue.test.ts b/apps/sim/lib/core/outbox/enqueue.test.ts index 3cda982d5eb..ef1d626e9d3 100644 --- a/apps/sim/lib/core/outbox/enqueue.test.ts +++ b/apps/sim/lib/core/outbox/enqueue.test.ts @@ -27,7 +27,14 @@ const IDLE_MINUTE = new Date('2026-09-16T12:34:45Z') const MAINTENANCE_MINUTE = new Date('2026-09-16T12:35:10Z') const INLINE_OUTPUT = { - result: { processed: 0, retried: 0, deadLettered: 0, leaseLost: 0, reaped: 0 }, + result: { + processed: 0, + retried: 0, + deadLettered: 0, + leaseLost: 0, + reaped: 0, + unloadedEventTypes: [], + }, recoveredDocuments: 0, reapedBackgroundWork: 0, } @@ -84,7 +91,14 @@ describe('outbox processor enqueue', () => { it('preserves synchronous processing for self-hosted deployments without Trigger', async () => { setEnvFlags({ isTriggerDevEnabled: false }) const output = { - result: { processed: 4, retried: 0, deadLettered: 0, leaseLost: 0, reaped: 0 }, + result: { + processed: 4, + retried: 0, + deadLettered: 0, + leaseLost: 0, + reaped: 0, + unloadedEventTypes: [], + }, recoveredDocuments: 2, reapedBackgroundWork: 1, } diff --git a/apps/sim/lib/core/outbox/processor.test.ts b/apps/sim/lib/core/outbox/processor.test.ts index c818129df29..4676a76503e 100644 --- a/apps/sim/lib/core/outbox/processor.test.ts +++ b/apps/sim/lib/core/outbox/processor.test.ts @@ -24,7 +24,14 @@ import { runOutboxProcessor } from '@/lib/core/outbox/processor' const mockProcessOutboxEvents = outboxServiceMockFns.mockProcessOutboxEvents describe('outbox processor recovery', () => { - const result = { processed: 5, retried: 1, deadLettered: 0, leaseLost: 0, reaped: 0 } + const result = { + processed: 5, + retried: 1, + deadLettered: 0, + leaseLost: 0, + reaped: 0, + unloadedEventTypes: [], + } beforeEach(() => { vi.resetAllMocks() @@ -86,4 +93,18 @@ describe('outbox processor recovery', () => { expect(mocks.recover).not.toHaveBeenCalled() expect(mocks.reap).not.toHaveBeenCalled() }) + + it('finishes maintenance, then fails the run naming event types whose handler module failed to load', async () => { + mockProcessOutboxEvents.mockResolvedValueOnce({ + ...result, + unloadedEventTypes: ['test.broken', 'test.broken-too'], + }) + + await expect(runOutboxProcessor()).rejects.toThrow( + 'Outbox handler modules failed to load; left pending: test.broken, test.broken-too' + ) + expect(mocks.recover).toHaveBeenCalledOnce() + expect(mocks.reap).toHaveBeenCalledOnce() + expect(mocks.prune).toHaveBeenCalledOnce() + }) }) diff --git a/apps/sim/lib/core/outbox/processor.ts b/apps/sim/lib/core/outbox/processor.ts index 37d27145a1e..1e5532869ee 100644 --- a/apps/sim/lib/core/outbox/processor.ts +++ b/apps/sim/lib/core/outbox/processor.ts @@ -71,5 +71,11 @@ export async function runOutboxProcessor(): Promise { prunedEvents, durationMs: Date.now() - startedAt, }) + /** Fail the run so a broken handler module stays as visible as the crash its static import caused. */ + if (result.unloadedEventTypes.length > 0) { + throw new Error( + `Outbox handler modules failed to load; left pending: ${result.unloadedEventTypes.join(', ')}` + ) + } return output } diff --git a/apps/sim/lib/core/outbox/service.test.ts b/apps/sim/lib/core/outbox/service.test.ts index 356e1693972..d801dd2cfe1 100644 --- a/apps/sim/lib/core/outbox/service.test.ts +++ b/apps/sim/lib/core/outbox/service.test.ts @@ -637,6 +637,7 @@ describe('processOutboxEvents — lazy handler groups', () => { processed: 1, retried: 0, deadLettered: 0, + unloadedEventTypes: ['test.flaky'], }) expect(healthyHandler).toHaveBeenCalledOnce() expect(claimedEventTypes()).not.toContain('test.flaky') @@ -646,7 +647,10 @@ describe('processOutboxEvents — lazy handler groups', () => { queuePendingEvents([makePendingRow({ eventType: 'test.flaky' })]) holdLease() - expect(await processOutboxEvents(groups)).toMatchObject({ processed: 1 }) + expect(await processOutboxEvents(groups)).toMatchObject({ + processed: 1, + unloadedEventTypes: [], + }) expect(recoveredHandler).toHaveBeenCalledOnce() }) diff --git a/apps/sim/lib/core/outbox/service.ts b/apps/sim/lib/core/outbox/service.ts index adcd2a22c32..2990714e39c 100644 --- a/apps/sim/lib/core/outbox/service.ts +++ b/apps/sim/lib/core/outbox/service.ts @@ -166,6 +166,8 @@ export interface ProcessOutboxResult { deadLettered: number leaseLost: number reaped: number + /** Ready event types left pending because their handler module failed to import. */ + unloadedEventTypes: string[] } export type ProcessSingleOutboxResult = @@ -457,7 +459,7 @@ export async function processOutboxEvents( reaped = await reapStuckProcessingRows() phase = 'discover' const readyTypes = await db.execute<{ eventType: string }>(readyEventTypesQuery(new Date())) - const { handlers, eligibleTypes } = await resolveOutboxHandlers( + const { handlers, eligibleTypes, unloadedEventTypes } = await resolveOutboxHandlers( handlerGroups, readyTypes.map(({ eventType }) => eventType) ) @@ -497,7 +499,7 @@ export async function processOutboxEvents( else retried++ } - return { processed, retried, deadLettered, leaseLost, reaped } + return { processed, retried, deadLettered, leaseLost, reaped, unloadedEventTypes } } catch (error) { logger.error('Outbox processing failed', { phase, @@ -517,13 +519,17 @@ export async function processOutboxEvents( * Imports the groups that serve any ready event type and returns the types to claim this run. A * group whose import fails leaves its event types unclaimed: unlike a missing handler, which * spends an attempt and eventually dead-letters, a failed import says nothing about the events, - * so they stay pending for a later run. Event types outside every group stay eligible and reach - * the missing-handler path. + * so they stay pending for a later run and are reported as `unloadedEventTypes`. Event types + * outside every group stay eligible and reach the missing-handler path. */ async function resolveOutboxHandlers( groups: readonly LazyOutboxHandlerGroup[], readyEventTypes: string[] -): Promise<{ handlers: OutboxHandlerRegistry; eligibleTypes: string[] }> { +): Promise<{ + handlers: OutboxHandlerRegistry + eligibleTypes: string[] + unloadedEventTypes: string[] +}> { const unavailableEventTypes = new Set() const ready = new Set(readyEventTypes) const dueGroups = groups.filter((group) => group.events.some((eventType) => ready.has(eventType))) @@ -547,6 +553,7 @@ async function resolveOutboxHandlers( return { handlers, eligibleTypes: readyEventTypes.filter((eventType) => !unavailableEventTypes.has(eventType)), + unloadedEventTypes: readyEventTypes.filter((eventType) => unavailableEventTypes.has(eventType)), } } From 725a3b3a5674327b0a98809f6b1625062762dff7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 00:38:29 -0700 Subject: [PATCH 4/5] improvement(outbox): import every registry event type from a light module; prove failed loads in Postgres --- apps/sim/lib/core/outbox/handlers.ts | 8 ++-- apps/sim/lib/core/outbox/retention.ts | 4 +- .../lib/core/outbox/service.integration.ts | 45 +++++++++++++++++++ .../connector-upload.integration.ts | 11 ++--- ...dormant-processing-recovery.integration.ts | 6 +-- .../storage-cleanup.integration.ts | 2 +- .../stored-document-recovery.integration.ts | 2 +- .../workspace-import.integration.ts | 2 +- .../connectors/sync-content-pass.test.ts | 1 - .../knowledge/connectors/sync-engine.test.ts | 1 - .../connectors/sync-persistence.test.ts | 1 - .../processing-outbox-handler.test.ts | 2 +- .../documents/processing-outbox-handler.ts | 8 ++-- .../documents/processing-recovery-event.ts | 1 + .../documents/processing-recovery.ts | 2 +- .../documents/storage-cleanup-event.ts | 1 + .../documents/storage-cleanup.test.ts | 2 +- .../knowledge/documents/storage-cleanup.ts | 2 +- .../documents/storage-upload.test.ts | 1 - .../lib/knowledge/documents/storage-upload.ts | 2 +- 20 files changed, 72 insertions(+), 32 deletions(-) create mode 100644 apps/sim/lib/knowledge/documents/processing-recovery-event.ts create mode 100644 apps/sim/lib/knowledge/documents/storage-cleanup-event.ts diff --git a/apps/sim/lib/core/outbox/handlers.ts b/apps/sim/lib/core/outbox/handlers.ts index b891c745843..1eb6d28d893 100644 --- a/apps/sim/lib/core/outbox/handlers.ts +++ b/apps/sim/lib/core/outbox/handlers.ts @@ -29,8 +29,8 @@ import { KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT, KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, } from '@/lib/knowledge/documents/processing-outbox-event' -import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery' -import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup' +import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' import { INBOX_CLEANUP_EVENT } from '@/lib/mothership/inbox/cleanup-event' import { ORGANIZATION_RESOURCE_CLEANUP_EVENT } from '@/lib/organizations/resource-cleanup-event' import { @@ -54,8 +54,8 @@ import { FORK_CONTENT_COPY_EVENT } from '@/ee/workspace-forking/application/cont /** * Every handler module the outbox processor serves, with the event types its handler map * registers. A module is imported only when one of its event types is due, so a run never loads - * the dependencies of handlers it will not call. Event types come from dependency-free modules - * (or ones the processor already loads), so naming them here costs nothing. + * the dependencies of handlers it will not call. Event types come from dependency-free modules, + * so naming them here costs nothing. * * `handlers.test.ts` checks every group against its module. Each `load` destructures its import * so the unused-export audit can see which export it reads; `(await import(x)).y` marks every diff --git a/apps/sim/lib/core/outbox/retention.ts b/apps/sim/lib/core/outbox/retention.ts index 9244ec36d82..da910352968 100644 --- a/apps/sim/lib/core/outbox/retention.ts +++ b/apps/sim/lib/core/outbox/retention.ts @@ -2,8 +2,8 @@ import { db } from '@sim/db' import { outboxEvent } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, asc, eq, inArray, lt, sql } from 'drizzle-orm' -import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery' -import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup' +import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' const logger = createLogger('OutboxRetention') diff --git a/apps/sim/lib/core/outbox/service.integration.ts b/apps/sim/lib/core/outbox/service.integration.ts index d1b77efc066..7e349750e95 100644 --- a/apps/sim/lib/core/outbox/service.integration.ts +++ b/apps/sim/lib/core/outbox/service.integration.ts @@ -220,6 +220,51 @@ describe('outbox scheduling in PostgreSQL', () => { expect(pending.availableAt.getTime()).toBeGreaterThan(Date.now()) }) + it('leaves the events of a handler module that fails to import untouched until it loads', async () => { + const [stranded] = await enqueue('test.outbox.unloadable', 1) + const [delivered] = await enqueue('test.outbox.loadable', 1) + const [before] = await db.select().from(outboxEvent).where(eq(outboxEvent.id, stranded.id)) + const unloadable = { + events: ['test.outbox.unloadable'], + load: vi + .fn() + .mockRejectedValueOnce(new Error('Cannot find module')) + .mockResolvedValue({ 'test.outbox.unloadable': async () => {} }), + } + const groups = [ + unloadable, + { + events: ['test.outbox.loadable'], + load: async () => ({ 'test.outbox.loadable': async () => {} }), + }, + ] + + expect(await processOutboxEvents(groups)).toMatchObject({ + processed: 1, + retried: 0, + deadLettered: 0, + unloadedEventTypes: ['test.outbox.unloadable'], + }) + const [pending] = await db.select().from(outboxEvent).where(eq(outboxEvent.id, stranded.id)) + expect(pending).toMatchObject({ + status: 'pending', + attempts: before.attempts, + lockedAt: null, + lastError: null, + processedAt: null, + availableAt: before.availableAt, + }) + const [completed] = await db.select().from(outboxEvent).where(eq(outboxEvent.id, delivered.id)) + expect(completed.status).toBe('completed') + + expect(await processOutboxEvents(groups)).toMatchObject({ + processed: 1, + unloadedEventTypes: [], + }) + const [recovered] = await db.select().from(outboxEvent).where(eq(outboxEvent.id, stranded.id)) + expect(recovered.status).toBe('completed') + }) + it('lets claims skip a locked head without hiding other rows of that type', async () => { const [locked, available] = await enqueue('test.outbox.locked', 2) const delivered: string[] = [] diff --git a/apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts b/apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts index a2831d2bf6c..b8940b35ff0 100644 --- a/apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/connector-upload.integration.ts @@ -32,6 +32,7 @@ import { import { stillHoldsSyncLock } from '@/lib/knowledge/connectors/sync-lock' import { addDocument, updateDocument } from '@/lib/knowledge/connectors/sync-persistence' import * as cleanup from '@/lib/knowledge/documents/storage-cleanup' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' import { claimKnowledgeUploadForAttachment, uploadKnowledgeArtifact, @@ -76,7 +77,7 @@ describe('connector upload crash recovery', () => { .select() .from(outboxEvent) .where( - sql`${outboxEvent.eventType} = ${cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT} AND ${outboxEvent.payload}::jsonb ->> 'documentId' = ${documentId}` + sql`${outboxEvent.eventType} = ${KNOWLEDGE_STORAGE_CLEANUP_EVENT} AND ${outboxEvent.payload}::jsonb ->> 'documentId' = ${documentId}` ) .limit(1) expect(event).toBeDefined() @@ -87,7 +88,7 @@ describe('connector upload crash recovery', () => { .where(eq(outboxEvent.id, event.id)) expect( await processOutboxEventById(event.id, { - [cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanup.cleanupKnowledgeStorage, + [KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanup.cleanupKnowledgeStorage, }) ).toBe('completed') } @@ -217,7 +218,7 @@ describe('connector upload crash recovery', () => { expect( await processOutboxEventById(pending[0].id, { - [cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanup.cleanupKnowledgeStorage, + [KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanup.cleanupKnowledgeStorage, }) ).toBe('completed') expect(await getFileMetadataByKeys([oldKey!], 'knowledge-base')).toEqual([]) @@ -308,7 +309,7 @@ describe('connector upload crash recovery', () => { .select() .from(outboxEvent) .where( - sql`${outboxEvent.eventType} = ${cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT} AND ${outboxEvent.payload}->>'key' = ${file.key}` + sql`${outboxEvent.eventType} = ${KNOWLEDGE_STORAGE_CLEANUP_EVENT} AND ${outboxEvent.payload}->>'key' = ${file.key}` ) .limit(1) events.push(event.id) @@ -330,7 +331,7 @@ describe('connector upload crash recovery', () => { .toBe(true) const handlers = { - [cleanup.KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanup.cleanupKnowledgeStorage, + [KNOWLEDGE_STORAGE_CLEANUP_EVENT]: cleanup.cleanupKnowledgeStorage, } expect(await processOutboxEventById(event.id, handlers)).toBe('pending') expect(await readFile(path.join(fixtureStorage.root, file.key), 'utf8')).toBe( diff --git a/apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts index 2ccf293562c..45de8a58344 100644 --- a/apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts @@ -36,10 +36,8 @@ import { } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' import { addDocument } from '@/lib/knowledge/connectors/sync-persistence' -import { - KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT, - recoverKnowledgeDocumentProcessing, -} from '@/lib/knowledge/documents/processing-recovery' +import { recoverKnowledgeDocumentProcessing } from '@/lib/knowledge/documents/processing-recovery' +import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' import { QUEUED_DISPATCH_GRACE_MS } from '@/lib/knowledge/documents/types' type FixtureIds = ReturnType diff --git a/apps/sim/lib/knowledge/__integration__/storage-cleanup.integration.ts b/apps/sim/lib/knowledge/__integration__/storage-cleanup.integration.ts index cd45fc96672..4351f9c0f2e 100644 --- a/apps/sim/lib/knowledge/__integration__/storage-cleanup.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/storage-cleanup.integration.ts @@ -34,8 +34,8 @@ import { hardDeleteDocuments } from '@/lib/knowledge/documents/service' import { cleanupKnowledgeStorage, enqueueKnowledgeStorageCleanup, - KNOWLEDGE_STORAGE_CLEANUP_EVENT, } from '@/lib/knowledge/documents/storage-cleanup' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' import * as storage from '@/lib/uploads/core/storage-service' import { deleteFileMetadataByIdentity, diff --git a/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts index 50b29de0fc2..f30de58f5f2 100644 --- a/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts @@ -88,10 +88,10 @@ import { import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { DOCUMENT_RECOVERY_BATCH_SIZE, - KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT, recoverKnowledgeDocumentProcessing, recoveryCandidatesQuery, } from '@/lib/knowledge/documents/processing-recovery' +import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' import { processDocumentAsync, processDocumentsWithQueue, diff --git a/apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts b/apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts index 113eaa599e0..7873ba02978 100644 --- a/apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts @@ -51,7 +51,7 @@ import { searchKnowledge } from '@/lib/knowledge/application/search' import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { createDocumentRecords, createSingleDocument } from '@/lib/knowledge/documents/service' -import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' import { uploadKnowledgeArtifact } from '@/lib/knowledge/documents/storage-upload' import { deleteWorkspaceFile, diff --git a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts index 85213894601..56f08610688 100644 --- a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts @@ -71,7 +71,6 @@ vi.mock('@/lib/uploads', () => uploadsMock) vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ - KNOWLEDGE_STORAGE_CLEANUP_EVENT: 'knowledge.document.storage.cleanup', enqueueKnowledgeStorageCleanup: hoisted.enqueueCleanup, isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), })) diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index dc375af379f..a9e18722987 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -89,7 +89,6 @@ vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) const bindings = vi.hoisted(() => new Map()) vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ - KNOWLEDGE_STORAGE_CLEANUP_EVENT: 'knowledge.document.storage.cleanup', enqueueKnowledgeStorageCleanup: mockEnqueueStorageCleanup, isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), })) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts index 3cad323a2dc..dab336926e4 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts @@ -22,7 +22,6 @@ vi.mock('@/lib/uploads', () => uploadsMock) vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ - KNOWLEDGE_STORAGE_CLEANUP_EVENT: 'knowledge.document.storage.cleanup', enqueueKnowledgeStorageCleanup: vi.fn(async () => { dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'cleanup-guard' }]) return ['cleanup-guard'] diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts index a3791db4a19..727b2e8256a 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts @@ -20,7 +20,7 @@ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attr import type { OutboxEventContext } from '@/lib/core/outbox/service' import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' -import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery' +import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' const mockGetKnowledgeDocument = knowledgeDocumentsServiceMockFns.mockGetKnowledgeDocument const mockProcessDocumentsWithQueue = knowledgeDocumentsServiceMockFns.mockProcessDocumentsWithQueue diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts index 57bf710d93d..cd129c36535 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts @@ -49,17 +49,15 @@ import { canScheduleDocumentProcessingQuotaContinuation, scheduleDocumentProcessingQuotaContinuation, } from '@/lib/knowledge/documents/processing-quota-continuation' -import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery' +import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' import { getKnowledgeDocument, type ProcessingOptions, processDocumentAsync, processDocumentsWithQueue, } from '@/lib/knowledge/documents/service' -import { - cleanupKnowledgeStorage, - KNOWLEDGE_STORAGE_CLEANUP_EVENT, -} from '@/lib/knowledge/documents/storage-cleanup' +import { cleanupKnowledgeStorage } from '@/lib/knowledge/documents/storage-cleanup' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' function requirePayloadRecord(payload: unknown): Record { if (!isRecordLike(payload)) { diff --git a/apps/sim/lib/knowledge/documents/processing-recovery-event.ts b/apps/sim/lib/knowledge/documents/processing-recovery-event.ts new file mode 100644 index 00000000000..88e159d6a81 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-recovery-event.ts @@ -0,0 +1 @@ +export const KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT = 'knowledge.document.processing.recover' diff --git a/apps/sim/lib/knowledge/documents/processing-recovery.ts b/apps/sim/lib/knowledge/documents/processing-recovery.ts index 0a713a20749..2d1e974e17f 100644 --- a/apps/sim/lib/knowledge/documents/processing-recovery.ts +++ b/apps/sim/lib/knowledge/documents/processing-recovery.ts @@ -18,6 +18,7 @@ import { createOrganizationDocumentProcessingBillingContext, createWorkspaceDocumentProcessingBillingContext, } from '@/lib/knowledge/documents/processing-payload' +import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' import { documentProcessingRecoveryCondition } from '@/lib/knowledge/documents/processing-recovery-policy' import { DOCUMENT_LIVENESS_BATCH_SIZE, @@ -28,7 +29,6 @@ import { const logger = createLogger('KnowledgeDocumentRecovery') -export const KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT = 'knowledge.document.processing.recover' export const DOCUMENT_RECOVERY_BATCH_SIZE = 200 const RECOVERY_RUNTIME_MS = 20_000 const MAX_RECOVERY_CANDIDATE_BATCHES = 4 diff --git a/apps/sim/lib/knowledge/documents/storage-cleanup-event.ts b/apps/sim/lib/knowledge/documents/storage-cleanup-event.ts new file mode 100644 index 00000000000..2c59ea5ec8e --- /dev/null +++ b/apps/sim/lib/knowledge/documents/storage-cleanup-event.ts @@ -0,0 +1 @@ +export const KNOWLEDGE_STORAGE_CLEANUP_EVENT = 'knowledge.document.storage.cleanup' diff --git a/apps/sim/lib/knowledge/documents/storage-cleanup.test.ts b/apps/sim/lib/knowledge/documents/storage-cleanup.test.ts index 711ee6e0f43..5807345fab6 100644 --- a/apps/sim/lib/knowledge/documents/storage-cleanup.test.ts +++ b/apps/sim/lib/knowledge/documents/storage-cleanup.test.ts @@ -14,8 +14,8 @@ vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) import { cleanupKnowledgeStorage, enqueueKnowledgeStorageCleanup, - KNOWLEDGE_STORAGE_CLEANUP_EVENT, } from '@/lib/knowledge/documents/storage-cleanup' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' const mockDeleteFile = storageServiceMockFns.mockDeleteFile const mockDeleteMetadata = uploadsMetadataMockFns.mockDeleteFileMetadataByIdentity diff --git a/apps/sim/lib/knowledge/documents/storage-cleanup.ts b/apps/sim/lib/knowledge/documents/storage-cleanup.ts index 024cdac6c47..05f25820892 100644 --- a/apps/sim/lib/knowledge/documents/storage-cleanup.ts +++ b/apps/sim/lib/knowledge/documents/storage-cleanup.ts @@ -12,6 +12,7 @@ import { } from '@/lib/core/resource-scope' import type { DbOrTx } from '@/lib/db/types' import { checkpointIo } from '@/lib/knowledge/documents/processing-checkpoint-io' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' import { deleteFile } from '@/lib/uploads/core/storage-service' import { deleteFileMetadataByIdentity, getFileMetadataByKeys } from '@/lib/uploads/server/metadata' import { headProviderObject, uploadStorageProvider } from '@/lib/uploads/upload-session/provider' @@ -20,7 +21,6 @@ import { extractStorageKey } from '@/lib/uploads/utils/file-utils' const logger = createLogger('KnowledgeStorageCleanup') const ENQUEUE_BATCH_SIZE = 100 const STORAGE_TIMEOUT_MS = 15_000 -export const KNOWLEDGE_STORAGE_CLEANUP_EVENT = 'knowledge.document.storage.cleanup' export interface KnowledgeStorageCleanupDocument extends ResourceOwner { id: string diff --git a/apps/sim/lib/knowledge/documents/storage-upload.test.ts b/apps/sim/lib/knowledge/documents/storage-upload.test.ts index 6df922a8d59..3e52632a18c 100644 --- a/apps/sim/lib/knowledge/documents/storage-upload.test.ts +++ b/apps/sim/lib/knowledge/documents/storage-upload.test.ts @@ -11,7 +11,6 @@ const mocks = vi.hoisted(() => ({ enqueue: vi.fn() })) vi.mock('@/lib/uploads', () => uploadsMock) vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock) vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ - KNOWLEDGE_STORAGE_CLEANUP_EVENT: 'knowledge.document.storage.cleanup', isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), enqueueKnowledgeStorageCleanup: mocks.enqueue, })) diff --git a/apps/sim/lib/knowledge/documents/storage-upload.ts b/apps/sim/lib/knowledge/documents/storage-upload.ts index 95536f3ecd3..69c5fd3b482 100644 --- a/apps/sim/lib/knowledge/documents/storage-upload.ts +++ b/apps/sim/lib/knowledge/documents/storage-upload.ts @@ -7,8 +7,8 @@ import type { DbTransaction } from '@/lib/db/types' import { enqueueKnowledgeStorageCleanup, isKnowledgeBaseOwnedStorageKey, - KNOWLEDGE_STORAGE_CLEANUP_EVENT, } from '@/lib/knowledge/documents/storage-cleanup' +import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' import { StorageService } from '@/lib/uploads' import { insertImmutableFileMetadata } from '@/lib/uploads/server/metadata' From dc055bc2c5317b39c28ee923d2f5a12f94ffa66b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 01:01:22 -0700 Subject: [PATCH 5/5] improvement(outbox): source enterprise and document-processing event types from dependency-free modules --- .../app/api/v1/admin/outbox/[id]/requeue/route.ts | 4 ++-- apps/sim/lib/admin/dashboard-credit-grant.test.ts | 1 - apps/sim/lib/admin/dashboard-organizations.test.ts | 1 - apps/sim/lib/admin/dashboard.ts | 2 +- apps/sim/lib/admin/invitation-operation.ts | 6 ++---- apps/sim/lib/billing/enterprise-outbox-events.ts | 5 +++++ apps/sim/lib/billing/enterprise-outbox.ts | 10 ++++------ apps/sim/lib/billing/enterprise-provisioning.ts | 14 +++++++------- apps/sim/lib/billing/webhooks/enterprise.ts | 12 +++++++----- apps/sim/lib/core/outbox/handlers.ts | 4 ++-- .../execution-archive-provenance.integration.ts | 2 +- .../stored-document-recovery.integration.ts | 6 ++---- .../workspace-import.integration.ts | 2 +- .../documents/document-processing-source.test.ts | 2 +- .../lib/knowledge/documents/processing-events.ts | 3 +++ .../knowledge/documents/processing-outbox-event.ts | 9 ++++----- .../documents/processing-outbox-handler.test.ts | 2 +- .../documents/processing-outbox-handler.ts | 4 ++-- scripts/check-unused-exports.baseline.json | 2 -- 19 files changed, 45 insertions(+), 46 deletions(-) create mode 100644 apps/sim/lib/billing/enterprise-outbox-events.ts create mode 100644 apps/sim/lib/knowledge/documents/processing-events.ts diff --git a/apps/sim/app/api/v1/admin/outbox/[id]/requeue/route.ts b/apps/sim/app/api/v1/admin/outbox/[id]/requeue/route.ts index e6c84765379..944c0b94036 100644 --- a/apps/sim/app/api/v1/admin/outbox/[id]/requeue/route.ts +++ b/apps/sim/app/api/v1/admin/outbox/[id]/requeue/route.ts @@ -6,11 +6,11 @@ import { and, eq, sql } from 'drizzle-orm' import { NextResponse } from 'next/server' import { adminV1RequeueOutboxEventContract } from '@/lib/api/contracts/v1/admin' import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' +import { enterpriseMetadataSyncPayloadSchema } from '@/lib/billing/enterprise-outbox' import { ENTERPRISE_METADATA_SYNC_EVENT_TYPE, ENTERPRISE_PROVISION_EVENT_TYPE, - enterpriseMetadataSyncPayloadSchema, -} from '@/lib/billing/enterprise-outbox' +} from '@/lib/billing/enterprise-outbox-events' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { withAdminAuthParams } from '@/app/api/v1/admin/middleware' diff --git a/apps/sim/lib/admin/dashboard-credit-grant.test.ts b/apps/sim/lib/admin/dashboard-credit-grant.test.ts index a9c576da62b..a3512962049 100644 --- a/apps/sim/lib/admin/dashboard-credit-grant.test.ts +++ b/apps/sim/lib/admin/dashboard-credit-grant.test.ts @@ -50,7 +50,6 @@ vi.mock('@/lib/billing/enterprise-provisioning', () => ({ getLatestEnterpriseProvisionings: vi.fn(async () => new Map()), })) vi.mock('@/lib/billing/enterprise-outbox', () => ({ - ENTERPRISE_METADATA_SYNC_EVENT_TYPE: 'stripe.sync-enterprise-metadata', resolveEnterpriseMetadataIntent: vi.fn(), })) vi.mock('@/lib/core/outbox/service', () => outboxServiceMock) diff --git a/apps/sim/lib/admin/dashboard-organizations.test.ts b/apps/sim/lib/admin/dashboard-organizations.test.ts index e19992587dc..57b494e1012 100644 --- a/apps/sim/lib/admin/dashboard-organizations.test.ts +++ b/apps/sim/lib/admin/dashboard-organizations.test.ts @@ -48,7 +48,6 @@ vi.mock('@/lib/billing/enterprise-provisioning', () => ({ getLatestEnterpriseProvisionings: vi.fn(async () => hoistedMocks.provisionings), })) vi.mock('@/lib/billing/enterprise-outbox', () => ({ - ENTERPRISE_METADATA_SYNC_EVENT_TYPE: 'stripe.sync-enterprise-metadata', enterpriseMetadataSyncPayloadSchema: { safeParse: vi.fn() }, resolveEnterpriseMetadataIntent: hoistedMocks.resolveMetadataIntent, })) diff --git a/apps/sim/lib/admin/dashboard.ts b/apps/sim/lib/admin/dashboard.ts index bdb62e65b72..83add513a13 100644 --- a/apps/sim/lib/admin/dashboard.ts +++ b/apps/sim/lib/admin/dashboard.ts @@ -44,10 +44,10 @@ import { } from '@/lib/billing/core/reporting-period' import { creditsToDollars, dollarsToCredits } from '@/lib/billing/credits/conversion' import { - ENTERPRISE_METADATA_SYNC_EVENT_TYPE, enterpriseMetadataSyncPayloadSchema, resolveEnterpriseMetadataIntent, } from '@/lib/billing/enterprise-outbox' +import { ENTERPRISE_METADATA_SYNC_EVENT_TYPE } from '@/lib/billing/enterprise-outbox-events' import { type EnterpriseProvisioningView, getLatestEnterpriseProvisionings, diff --git a/apps/sim/lib/admin/invitation-operation.ts b/apps/sim/lib/admin/invitation-operation.ts index 61037f04368..05597841102 100644 --- a/apps/sim/lib/admin/invitation-operation.ts +++ b/apps/sim/lib/admin/invitation-operation.ts @@ -5,10 +5,8 @@ import { normalizeEmail } from '@sim/utils/string' import { and, count, eq, inArray, sql } from 'drizzle-orm' import { z } from 'zod' import { ADMIN_INVITATION_OPERATION_EVENT_TYPE } from '@/lib/admin/invitation-operation-event' -import { - ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE, - enterpriseInvitePeoplePayloadSchema, -} from '@/lib/billing/enterprise-outbox' +import { enterpriseInvitePeoplePayloadSchema } from '@/lib/billing/enterprise-outbox' +import { ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE } from '@/lib/billing/enterprise-outbox-events' import { acquireOrganizationMutationLock } from '@/lib/billing/organizations/membership' import { deferOutboxHandler, diff --git a/apps/sim/lib/billing/enterprise-outbox-events.ts b/apps/sim/lib/billing/enterprise-outbox-events.ts new file mode 100644 index 00000000000..c224eea8584 --- /dev/null +++ b/apps/sim/lib/billing/enterprise-outbox-events.ts @@ -0,0 +1,5 @@ +export const ENTERPRISE_PROVISION_EVENT_TYPE = 'stripe.provision-enterprise' +export const ENTERPRISE_METADATA_SYNC_EVENT_TYPE = 'stripe.sync-enterprise-metadata' +export const ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE = 'enterprise.move-workspace' +export const ENTERPRISE_MEMBER_RECONCILIATION_EVENT_TYPE = 'enterprise.reconcile-members' +export const ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE = 'enterprise.invite-people' diff --git a/apps/sim/lib/billing/enterprise-outbox.ts b/apps/sim/lib/billing/enterprise-outbox.ts index 3eda75002d3..19087e30d8e 100644 --- a/apps/sim/lib/billing/enterprise-outbox.ts +++ b/apps/sim/lib/billing/enterprise-outbox.ts @@ -4,16 +4,14 @@ import { and, desc, eq, sql } from 'drizzle-orm' import type Stripe from 'stripe' import { z } from 'zod' import { MAX_BILLING_CONCURRENCY_LIMIT } from '@/lib/billing/concurrency-defaults' +import { + ENTERPRISE_METADATA_SYNC_EVENT_TYPE, + ENTERPRISE_PROVISION_EVENT_TYPE, +} from '@/lib/billing/enterprise-outbox-events' import { MAX_WORKFLOW_EXECUTION_TIMEOUT_SECONDS } from '@/lib/billing/execution-timeout-defaults' import type { DbOrTx } from '@/lib/db/types' import { MAX_INVITE_EMAILS } from '@/lib/invitations/limits' -export const ENTERPRISE_PROVISION_EVENT_TYPE = 'stripe.provision-enterprise' -export const ENTERPRISE_METADATA_SYNC_EVENT_TYPE = 'stripe.sync-enterprise-metadata' -export const ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE = 'enterprise.move-workspace' -export const ENTERPRISE_MEMBER_RECONCILIATION_EVENT_TYPE = 'enterprise.reconcile-members' -export const ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE = 'enterprise.invite-people' - const nonnegativeInteger = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER) export const enterpriseProvisionRequestSchema = z.object({ diff --git a/apps/sim/lib/billing/enterprise-provisioning.ts b/apps/sim/lib/billing/enterprise-provisioning.ts index 06a82a801f9..bb1d5dce165 100644 --- a/apps/sim/lib/billing/enterprise-provisioning.ts +++ b/apps/sim/lib/billing/enterprise-provisioning.ts @@ -42,11 +42,6 @@ import { import { creditsToDollars, dollarsToCredits } from '@/lib/billing/credits/conversion' import { deriveEnterpriseOperationStatus, - ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE, - ENTERPRISE_MEMBER_RECONCILIATION_EVENT_TYPE, - ENTERPRISE_METADATA_SYNC_EVENT_TYPE, - ENTERPRISE_PROVISION_EVENT_TYPE, - ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE, type EnterpriseInvitePeoplePayload, type EnterpriseMetadataSyncPayload, type EnterpriseOperationStatus, @@ -61,6 +56,13 @@ import { enterpriseWorkspaceMovePayloadSchema, parseEnterpriseProvisionPayload, } from '@/lib/billing/enterprise-outbox' +import { + ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE, + ENTERPRISE_MEMBER_RECONCILIATION_EVENT_TYPE, + ENTERPRISE_METADATA_SYNC_EVENT_TYPE, + ENTERPRISE_PROVISION_EVENT_TYPE, + ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE, +} from '@/lib/billing/enterprise-outbox-events' import { parseWorkflowExecutionTimeoutSeconds, resolveEnterpriseWorkflowExecutionTimeoutFallbackSeconds, @@ -3211,5 +3213,3 @@ export async function getLatestEnterpriseProvisionings( } return result } - -export { ENTERPRISE_METADATA_SYNC_EVENT_TYPE, ENTERPRISE_PROVISION_EVENT_TYPE } diff --git a/apps/sim/lib/billing/webhooks/enterprise.ts b/apps/sim/lib/billing/webhooks/enterprise.ts index e66807b7afd..abe549dfaf2 100644 --- a/apps/sim/lib/billing/webhooks/enterprise.ts +++ b/apps/sim/lib/billing/webhooks/enterprise.ts @@ -14,11 +14,6 @@ import { } from '@/lib/auth/security-policy' import { deriveEnterpriseCreditLimits } from '@/lib/billing/enterprise-credit-limits' import { - ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE, - ENTERPRISE_MEMBER_RECONCILIATION_EVENT_TYPE, - ENTERPRISE_METADATA_SYNC_EVENT_TYPE, - ENTERPRISE_PROVISION_EVENT_TYPE, - ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE, type EnterpriseProvisionPayload, enterpriseMetadataDeliveryIsVerified, enterpriseMetadataIntentMatchesStripeSubscription, @@ -26,6 +21,13 @@ import { enterpriseOperationMatchesStripeSubscription, parseEnterpriseProvisionPayload, } from '@/lib/billing/enterprise-outbox' +import { + ENTERPRISE_INVITE_PEOPLE_EVENT_TYPE, + ENTERPRISE_MEMBER_RECONCILIATION_EVENT_TYPE, + ENTERPRISE_METADATA_SYNC_EVENT_TYPE, + ENTERPRISE_PROVISION_EVENT_TYPE, + ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE, +} from '@/lib/billing/enterprise-outbox-events' import { getEnterpriseIssuanceSeatRequirement } from '@/lib/billing/enterprise-provisioning' import { acquireOrganizationMutationLock } from '@/lib/billing/organizations/membership' import { requireStripeClient } from '@/lib/billing/stripe-client' diff --git a/apps/sim/lib/core/outbox/handlers.ts b/apps/sim/lib/core/outbox/handlers.ts index 1eb6d28d893..73bc7401da5 100644 --- a/apps/sim/lib/core/outbox/handlers.ts +++ b/apps/sim/lib/core/outbox/handlers.ts @@ -6,7 +6,7 @@ import { ENTERPRISE_METADATA_SYNC_EVENT_TYPE, ENTERPRISE_PROVISION_EVENT_TYPE, ENTERPRISE_WORKSPACE_MOVE_EVENT_TYPE, -} from '@/lib/billing/enterprise-outbox' +} from '@/lib/billing/enterprise-outbox-events' import { ENTERPRISE_OWNER_ACTIVATION_EVENT_TYPE, ENTERPRISE_OWNER_CLAIM_EVENT_TYPE, @@ -28,7 +28,7 @@ import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/do import { KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT, KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, -} from '@/lib/knowledge/documents/processing-outbox-event' +} from '@/lib/knowledge/documents/processing-events' import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' import { INBOX_CLEANUP_EVENT } from '@/lib/mothership/inbox/cleanup-event' diff --git a/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts b/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts index b72f5b74471..3757c597836 100644 --- a/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts @@ -77,7 +77,7 @@ import { import { addWorkspaceFilesToKnowledgeBase } from '@/lib/knowledge/application/add-workspace-files' import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' import { searchKnowledge } from '@/lib/knowledge/application/search' -import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' +import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-events' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { createSingleDocument } from '@/lib/knowledge/documents/service' import { loadKnowledgeDocumentSecretRegistry } from '@/lib/knowledge/secret-provenance' diff --git a/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts index f30de58f5f2..9dc36a3698a 100644 --- a/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts @@ -81,10 +81,8 @@ import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' import { addDocument } from '@/lib/knowledge/connectors/sync-persistence' import { sweepStuckDocuments } from '@/lib/knowledge/connectors/sync-primitives' import { DEFERRED_RETRY_LOST_ERROR } from '@/lib/knowledge/documents/deferred-retry-check' -import { - enqueueKnowledgeDocumentProcessing, - KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT, -} from '@/lib/knowledge/documents/processing-outbox-event' +import { KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT } from '@/lib/knowledge/documents/processing-events' +import { enqueueKnowledgeDocumentProcessing } from '@/lib/knowledge/documents/processing-outbox-event' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { DOCUMENT_RECOVERY_BATCH_SIZE, diff --git a/apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts b/apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts index 7873ba02978..c27bdf44438 100644 --- a/apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/workspace-import.integration.ts @@ -48,7 +48,7 @@ import { import { addWorkspaceFilesToKnowledgeBase } from '@/lib/knowledge/application/add-workspace-files' import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' import { searchKnowledge } from '@/lib/knowledge/application/search' -import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' +import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-events' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { createDocumentRecords, createSingleDocument } from '@/lib/knowledge/documents/service' import { KNOWLEDGE_STORAGE_CLEANUP_EVENT } from '@/lib/knowledge/documents/storage-cleanup-event' diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 6bb0739a20c..445286074c6 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -76,7 +76,7 @@ import { UsageLimitDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-event' -import { KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' +import { KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT } from '@/lib/knowledge/documents/processing-events' import { processDocumentAsync, processDocumentsWithQueue } from '@/lib/knowledge/documents/service' import { MAX_PROCESSING_ATTEMPTS } from '@/lib/knowledge/documents/types' diff --git a/apps/sim/lib/knowledge/documents/processing-events.ts b/apps/sim/lib/knowledge/documents/processing-events.ts new file mode 100644 index 00000000000..707f7c19f93 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-events.ts @@ -0,0 +1,3 @@ +export const KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT = 'knowledge.document.processing.dispatch' +export const KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT = + 'knowledge.document.deferred-retry-check' diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-event.ts b/apps/sim/lib/knowledge/documents/processing-outbox-event.ts index 03d632076a2..9b9b382b515 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-event.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-event.ts @@ -1,13 +1,15 @@ import type { db } from '@sim/db' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import { enqueueOutboxEvent } from '@/lib/core/outbox/service' +import { + KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT, + KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, +} from '@/lib/knowledge/documents/processing-events' import type { DocumentProcessingLane } from '@/lib/knowledge/documents/processing-payload' import type { DocumentProcessingSnapshot } from '@/lib/knowledge/documents/processing-recovery-queue' import type { ProcessingOptions } from '@/lib/knowledge/documents/service' import { QUEUED_DISPATCH_GRACE_MS } from '@/lib/knowledge/documents/types' -export const KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT = 'knowledge.document.processing.dispatch' - export interface KnowledgeDocumentProcessingOutboxPayload { knowledgeBaseId: string documentId: string @@ -28,9 +30,6 @@ export function enqueueKnowledgeDocumentProcessing( return enqueueOutboxEvent(executor, KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, payload) } -export const KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT = - 'knowledge.document.deferred-retry-check' - /** Thrown failures only: every wait in `checkDeferredDocumentRetry` defers without spending one. */ export const DEFERRED_RETRY_CHECK_MAX_ATTEMPTS = 5 diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts index 727b2e8256a..55bbbb2fdbb 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts @@ -18,7 +18,7 @@ vi.mock('@/lib/knowledge/documents/processing-claim', () => ({ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { OutboxEventContext } from '@/lib/core/outbox/service' -import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' +import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-events' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { KNOWLEDGE_DOCUMENT_RECOVERY_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-recovery-event' diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts index cd129c36535..0d11ad69612 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts @@ -33,8 +33,8 @@ import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/do import { KNOWLEDGE_DOCUMENT_DEFERRED_RETRY_CHECK_EVENT, KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, - type KnowledgeDocumentProcessingOutboxPayload, -} from '@/lib/knowledge/documents/processing-outbox-event' +} from '@/lib/knowledge/documents/processing-events' +import type { KnowledgeDocumentProcessingOutboxPayload } from '@/lib/knowledge/documents/processing-outbox-event' import { assertDocumentProcessingPayload, resolveDocumentProcessingLane, diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index 6d11aa2dbc4..e19b7d75b75 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -4201,8 +4201,6 @@ "apps/sim/lib/billing/enterprise-owner-claim.ts#ENTERPRISE_OWNER_CLAIM_EXPIRY_DAYS", "apps/sim/lib/billing/enterprise-owner-claim.ts#enterpriseOwnerClaimPayloadSchema", "apps/sim/lib/billing/enterprise-owner-claim.ts#getEnterpriseOwnerClaimView", - "apps/sim/lib/billing/enterprise-provisioning.ts#ENTERPRISE_METADATA_SYNC_EVENT_TYPE", - "apps/sim/lib/billing/enterprise-provisioning.ts#ENTERPRISE_PROVISION_EVENT_TYPE", "apps/sim/lib/billing/enterprise-provisioning.ts#EnterpriseFollowUpJobKind", "apps/sim/lib/billing/enterprise-provisioning.ts#moveEnterpriseWorkspace", "apps/sim/lib/billing/index.ts#canEditLimit",