From 2cf565a7809bb1aa3f9dfc66ee13a73f12c7452b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 19:13:06 -0700 Subject: [PATCH 1/3] fix(audit): harden data drains and expand security event coverage --- .../docs/platform/enterprise/audit-logs.mdx | 24 +- .../docs/platform/enterprise/data-drains.mdx | 24 +- .../app/api/audit-logs/export/route.test.ts | 118 ---- apps/sim/app/api/audit-logs/export/route.ts | 206 ++----- .../sim/app/api/auth/forget-password/route.ts | 25 +- .../auth/sso/providers/[providerId]/route.ts | 16 +- apps/sim/app/api/auth/sso/register/route.ts | 35 ++ apps/sim/background/run-data-drain.ts | 18 +- apps/sim/lib/api/contracts/audit-logs.ts | 7 +- .../authorized-audit-log-use-case.ts | 28 +- .../application/export-audit-logs.ts | 86 +++ .../lib/audit-logs/application/operations.ts | 18 +- apps/sim/lib/audit-logs/export.integration.ts | 120 ++++ apps/sim/lib/audit-logs/query.integration.ts | 81 +++ apps/sim/lib/audit-logs/query.ts | 25 +- apps/sim/lib/auth/auth.ts | 418 ++++++++------ .../lib/auth/lifecycle-audit.integration.ts | 387 +++++++++++++ apps/sim/lib/auth/lifecycle-audit.ts | 368 ++++++++++++ .../core/security/input-validation.server.ts | 24 +- .../lib/data-drains/application/use-cases.ts | 23 +- .../data-drains/destinations/azure_blob.ts | 8 +- .../lib/data-drains/destinations/bigquery.ts | 5 +- .../data-drains/destinations/datadog.test.ts | 66 ++- .../lib/data-drains/destinations/datadog.ts | 82 +-- apps/sim/lib/data-drains/destinations/gcs.ts | 5 +- .../destinations/s3-transport.test.ts | 140 +++++ apps/sim/lib/data-drains/destinations/s3.ts | 118 +++- .../destinations/snowflake.test.ts | 5 +- .../lib/data-drains/destinations/snowflake.ts | 6 +- .../lib/data-drains/destinations/webhook.ts | 6 +- apps/sim/lib/data-drains/dispatcher.test.ts | 58 -- apps/sim/lib/data-drains/dispatcher.ts | 190 ++++--- apps/sim/lib/data-drains/enqueue.ts | 31 + apps/sim/lib/data-drains/errors.ts | 64 +++ apps/sim/lib/data-drains/limits.ts | 10 + .../lib/data-drains/service.integration.ts | 531 ++++++++++++++++++ apps/sim/lib/data-drains/service.test.ts | 140 ----- apps/sim/lib/data-drains/service.ts | 286 +++++++--- .../sim/lib/data-drains/sources/audit-logs.ts | 51 +- .../lib/data-drains/sources/copilot-chats.ts | 122 ++-- .../lib/data-drains/sources/copilot-runs.ts | 59 +- .../sources/coverage.integration.ts | 196 +++++++ apps/sim/lib/data-drains/sources/helpers.ts | 60 +- apps/sim/lib/data-drains/sources/job-logs.ts | 46 +- .../lib/data-drains/sources/workflow-logs.ts | 46 +- .../lib/logs/execution/trace-store.test.ts | 36 ++ apps/sim/lib/logs/execution/trace-store.ts | 25 +- .../application/row-audit.integration.ts | 175 ++++++ apps/sim/lib/table/application/rows.ts | 74 +++ .../lib/workflows/deployment-outbox.test.ts | 23 +- apps/sim/lib/workflows/deployment-outbox.ts | 4 +- packages/audit/src/types.ts | 10 + packages/db/schema.ts | 2 +- packages/testing/src/mocks/audit.mock.ts | 10 + vitest.shared.ts | 1 + 55 files changed, 3613 insertions(+), 1129 deletions(-) delete mode 100644 apps/sim/app/api/audit-logs/export/route.test.ts create mode 100644 apps/sim/lib/audit-logs/application/export-audit-logs.ts create mode 100644 apps/sim/lib/audit-logs/export.integration.ts create mode 100644 apps/sim/lib/audit-logs/query.integration.ts create mode 100644 apps/sim/lib/auth/lifecycle-audit.integration.ts create mode 100644 apps/sim/lib/auth/lifecycle-audit.ts create mode 100644 apps/sim/lib/data-drains/destinations/s3-transport.test.ts delete mode 100644 apps/sim/lib/data-drains/dispatcher.test.ts create mode 100644 apps/sim/lib/data-drains/enqueue.ts create mode 100644 apps/sim/lib/data-drains/errors.ts create mode 100644 apps/sim/lib/data-drains/limits.ts create mode 100644 apps/sim/lib/data-drains/service.integration.ts delete mode 100644 apps/sim/lib/data-drains/service.test.ts create mode 100644 apps/sim/lib/data-drains/sources/coverage.integration.ts create mode 100644 apps/sim/lib/table/application/row-audit.integration.ts diff --git a/apps/docs/content/docs/platform/enterprise/audit-logs.mdx b/apps/docs/content/docs/platform/enterprise/audit-logs.mdx index 089d8b46190..f30016c0e55 100644 --- a/apps/docs/content/docs/platform/enterprise/audit-logs.mdx +++ b/apps/docs/content/docs/platform/enterprise/audit-logs.mdx @@ -1,6 +1,6 @@ --- title: Audit Logs -description: Track every action taken across your organization's workspaces +description: Track configuration and security events across your organization's workspaces --- import { FAQ } from '@/components/ui/faq' @@ -44,7 +44,7 @@ X-API-Key: | `resourceType` | string | Filter by resource type (e.g. `workflow`) | | `resourceId` | string | Filter by a specific resource ID | | `workspaceId` | string | Filter by workspace | -| `actorId` | string | Filter by user ID. For organization-wide filters, the actor must be a current or former org member; workspace-scoped logs can also include external workspace members. | +| `actorId` | string | Filter by a current organization member’s user ID. For a departed-actor filter, use the CSV export with `includeDeparted=true`. | | `startDate` | string | ISO 8601 date — return logs on or after this date | | `endDate` | string | ISO 8601 date — return logs on or before this date | | `includeDeparted` | boolean | Include logs from members who have since left the organization (default `false`) | @@ -84,7 +84,7 @@ X-API-Key: Paginate by passing the `nextCursor` value as the `cursor` parameter in the next request. When `nextCursor` is absent, you have reached the last page. Each entry also includes `actorName`; `metadata` is an arbitrary per-action JSON object. The `limits` object reports your current rate-limit and usage status. -The API accepts both personal and workspace-scoped API keys. Rate limits apply — the response includes `X-RateLimit-*` headers with your current limit and remaining quota. +The organization audit API requires a personal API key belonging to an organization owner or admin. Workspace API keys are refused. Rate limits apply — the response includes `X-RateLimit-*` headers with your current limit and remaining quota. --- @@ -94,6 +94,7 @@ Audit log events follow a `resource.action` naming pattern. The table below list | Category | Example events | |----------|---------------| +| **Authentication** | `auth.login_succeeded`, `auth.login_failed`, `auth.logout`, `auth.impersonation_started`, `auth.impersonation_ended`, `auth.sessions_revoked` | | **Workflows** | `workflow.created`, `workflow.deleted`, `workflow.deployed`, `workflow.locked` | | **Workspaces** | `workspace.created`, `workspace.updated`, `workspace.deleted` | | **Members** | `member.invited`, `member.removed`, `member.role_changed` | @@ -103,7 +104,14 @@ Audit log events follow a `resource.action` naming pattern. The table below list | **Tables** | `table.created`, `table.updated`, `table.deleted` | | **API keys** | `api_key.created`, `api_key.revoked` | | **Credentials** | `credential.created`, `credential.deleted`, `oauth.disconnected` | -| **Organization** | `organization.updated`, `org_member.added`, `org_member.role_changed` | +| **Organization** | `organization.updated`, `org_member.added`, `org_member.role_changed`, `organization.sso_provider.updated` | +| **Exports and drains** | `audit_logs.exported`, `data_drain.created`, `data_drain.updated`, `data_drain.ran`, `data_drain.tested` | + +Authentication failures and password-reset requests use an anonymous actor and identify the target account separately. Impersonation and administrative session revocations identify the operator and the target organization. Table-row mutations record operation summaries and counts without copying row contents. + +Most audit writes are best effort: a database outage or process loss can prevent persistence even when the underlying action succeeds. Deployment activation uses an awaited, idempotent audit insert before its outbox checkpoint. Audit logs are not a complete tool-invocation stream or a transactionally guaranteed ledger for every operation. + +Organization visibility depends on current workspace ownership or stored organization attribution. Older workspace events without organization metadata may no longer appear in organization queries after the workspace is physically deleted. Workspace invitation events include whether the invite is for an internal organization member or an external workspace member in their metadata. External workspace members can appear as actors on workspace-scoped events, and appear in the roster with an External label without becoming organization members. @@ -116,11 +124,11 @@ Workspace invitation events include whether the invite is for an internal organi }, { question: "Are audit logs tamper-proof?", - answer: "Audit log entries are append-only and cannot be modified or deleted through the Sim interface or API. They represent a reliable record of actions taken in your organization." + answer: "The Sim interface and API do not provide audit-entry edits or deletion. Database administrators can still alter stored records, and entries are not cryptographically sealed. Use a data drain to archive them under your own storage and retention controls." }, { question: "Can I export audit logs?", - answer: "Yes. Use the API to export logs programmatically. Paginate through all records using the cursor parameter and store them in your own data warehouse or SIEM." + answer: "Yes. Download CSV from the audit viewer or paginate the API for your SIEM. CSV includes event, workspace, resource and actor IDs, IP address, user agent, surface and metadata. Organization administrators can include departed actors for historical investigations. CSV downloads are recorded in the audit trail and capped at 10,000 rows or 64 MB; the X-Export-Truncated header indicates a partial export. For continuous delivery, use an audit-log data drain." }, { question: "Are logs scoped to a single workspace or the whole organization?", @@ -128,11 +136,11 @@ Workspace invitation events include whether the invite is for an internal organi }, { question: "What information is included in each log entry?", - answer: "Each entry includes the event type, a description, the actor's name and email, the affected resource, the workspace, and a timestamp. IP addresses and user agents are not exposed through the API." + answer: "Each entry includes the event type, a description, the actor's name and email, the affected resource, the workspace, and a timestamp. The JSON API omits IP addresses and user agents; authorized CSV exports and audit-log drains include them." }, { question: "Can I filter logs by a specific user?", - answer: "Yes. Pass the actorId query parameter to filter logs by a specific user. Organization-wide actor filters require the actor to be a current or former member of your organization. Workspace-scoped logs may also include external workspace members who acted inside a workspace without joining the organization." + answer: "Yes. The JSON API accepts actorId for current organization members. CSV exports also support departed actors when includeDeparted=true. Workspace-scoped logs may include external workspace members who acted inside a workspace without joining the organization." } ]} /> diff --git a/apps/docs/content/docs/platform/enterprise/data-drains.mdx b/apps/docs/content/docs/platform/enterprise/data-drains.mdx index 44bb8d617a7..41b61a04e40 100644 --- a/apps/docs/content/docs/platform/enterprise/data-drains.mdx +++ b/apps/docs/content/docs/platform/enterprise/data-drains.mdx @@ -6,7 +6,7 @@ description: Continuously export workflow logs, audit logs, and Chat data to you import { FAQ } from '@/components/ui/faq' import { Image } from '@/components/ui/image' -Data Drains let organization owners and admins on Enterprise plans continuously export Sim data to a destination they control — a customer-owned S3 bucket, Google Cloud Storage bucket, Azure Blob container, BigQuery table, Snowflake table, Datadog logs intake, or an HTTPS webhook. A drain runs on a schedule, picks up only new rows since its last successful run, and writes them to the destination. Viewing drain configuration and run history is restricted to owners and admins as well, since destinations expose internal bucket names, table identifiers, and webhook URLs. +Data Drains let organization owners and admins on Enterprise plans continuously export Sim data to a destination they control — a customer-owned S3 bucket, Google Cloud Storage bucket, Azure Blob container, BigQuery table, Snowflake table, Datadog logs intake, or an HTTPS webhook. A drain runs on a schedule, picks up rows after its last acknowledged delivery, and writes them to the destination. Viewing drain configuration and run history is restricted to owners and admins as well, since destinations expose internal bucket names, table identifiers, and webhook URLs. Drains are independent of [Data Retention](/platform/enterprise/data-retention) but designed to compose with it — see [Pairing with Data Retention](#pairing-with-data-retention) below. @@ -38,12 +38,12 @@ A drain exports exactly one source. To export multiple sources, create multiple | **Workflow logs** | Workflow execution records (one row per execution, only after the run reaches a terminal state). | | **Job logs** | Background job records (deployed APIs, schedules, webhooks). Only terminal-state rows are exported. | | **Audit logs** | Organization- and workspace-scoped audit events — logins, permission changes, resource creation/deletion, drain configuration changes. | -| **Chats** | Chat history. | -| **Chat runs** | Chat run records (terminal state only). | +| **Chats** | Chat snapshots, including organization-owned chats outside a workspace. | +| **Chat runs** | Chat run records (terminal state only), with safe correlation context. Internal controller tokens and billing recovery receipts are excluded. | Each row is delivered as a single line of NDJSON. The shape of each row is part of the public schema and stable across versions; every row carries an `id` field that downstream consumers can use to dedupe. -Drains select new rows by their creation cursor and deliver them at least once. A failed run can deliver a row again, so consumers should deduplicate by its `id`. Mutable fields on **Chats** (messages, title, `lastSeenAt`) are a point-in-time snapshot and won't be re-emitted if the chat is later updated. Treat the export as append-only and reconstitute current state from your own system of record if you need it. +Workflow, job, and Chat run drains advance by their terminal timestamp; audit and Chat drains advance by creation time. A new drain exports all retained history. Reads use the primary database and leave a five-minute stability window before selecting new records. This reduces late-commit gaps but is not a change-data-capture guarantee. Drains deliver rows at least once. A failed run can deliver a row again, so consumers should deduplicate by its `id`. Mutable fields on **Chats** (messages, title, `lastSeenAt`) are a point-in-time snapshot and won't be re-emitted if the chat is later updated. Treat the export as append-only and reconstitute current state from your own system of record if you need it. --- @@ -111,7 +111,7 @@ Inserts each row into a target VARIANT column via the Snowflake SQL API v2 with - **Role** *(optional)* — Snowflake role to assume. - **Private key (PEM)** — PKCS8-encoded RSA private key. Register the matching public key on the Snowflake user via `ALTER USER ... SET RSA_PUBLIC_KEY = '...'`. -Each chunk becomes a single `INSERT INTO "DB"."SCHEMA"."TABLE" ("col") VALUES (PARSE_JSON(?)), ...` with one TEXT binding per row. Identifiers are quoted to preserve case. The destination handles Snowflake's async 202-then-poll pattern transparently. Per-row JSON payloads are capped at 16 MB to match Snowflake's VARIANT limit. +Each chunk becomes a single `INSERT INTO "DB"."SCHEMA"."TABLE" ("col") SELECT PARSE_JSON(column1) FROM VALUES (?), ...` with one TEXT binding per row. Identifiers are quoted to preserve case. The destination handles Snowflake's async 202-then-poll pattern transparently. Per-row JSON payloads are capped at 16 MB to match Snowflake's VARIANT limit. ### Datadog Logs @@ -122,7 +122,7 @@ POSTs each row as a log entry to Datadog's v2 logs intake. - **Tags** *(optional)* — comma-separated `ddtags` appended to every entry alongside auto-injected `sim_drain_id:`, `sim_run_id:`, and `sim_source:` tags. - **API key** — a Datadog API key (not an Application key) with logs-write permission. -Top-level row fields are auto-indexed as Datadog log attributes. The reserved fields `ddsource`, `service`, `ddtags`, and `message` are always set by Sim and override anything in the row. Payloads above 1 KB are gzip-compressed. Enforced limits match Datadog's intake: 5 MB per request (post-compression), 1000 entries per request, 1 MB per entry. +Top-level row fields are auto-indexed as Datadog log attributes. The reserved fields `ddsource`, `service`, `ddtags`, and `message` are always set by Sim and override anything in the row. Payloads above 1 KB are gzip-compressed. Enforced limits match Datadog's intake: 5 MB of uncompressed JSON per request, 1000 entries per request, and 1 MB per entry. Expanded log entries are split into bounded requests before compression. ### HTTPS Webhook @@ -168,9 +168,11 @@ You can also disable a drain with the **Enabled** toggle (it stops running but i ## Delivery semantics -Drains use an **opaque cursor** that advances only on full success. If a delivery fails partway through a run, the cursor is unchanged and the next run replays from the last successful position. +Drains use an **opaque cursor** that checkpoints each acknowledged chunk. If a later chunk fails, the next run resumes after the last acknowledged checkpoint. A crash between remote delivery and checkpointing can replay a chunk. -This is **at-least-once delivery**. Combined with the `id` field on every row and the `Idempotency-Key` header on every webhook chunk, downstream systems can dedupe deterministically. +A database claim prevents overlapping runs for the same drain and fences stale workers. Each run is bounded to 10,000 rows, 100 chunks, 100 MB of NDJSON, and a 20-minute work window, with a 25-minute hard deadline. A queued run that reaches a window limit schedules a continuation. Chunks are capped at 4 MB; individual database records, chat transcripts, and execution archives are capped at 1 MB before materialization. Oversized records fail visibly without advancing past them. Missing execution archives also fail rather than reporting an incomplete record as delivered. + +This is **at-least-once delivery**. Deduplicate by the source row’s `id`. The webhook `Idempotency-Key` protects retries of a chunk within one run; a replay in a later run has a new run ID. A destination that partially accepts a request may also repeat previously accepted rows on retry. The **last 10 runs** for each drain are visible by expanding its row in the settings page, with status, row count, bytes written, destination locator (`s3://...` or webhook URL), and the error message if it failed. @@ -180,7 +182,7 @@ The **last 10 runs** for each drain are visible by expanding its row in the sett - Destination credentials are encrypted at rest using the same key-rotation–aware encryption that protects OAuth tokens. - Credentials are **never** returned by the Sim API after creation. Updates accept new credentials; omitting them leaves the existing encrypted blob in place. -- Webhook URLs are SSRF-validated: HTTPS-only, no private/loopback/metadata IPs, with the resolved IP pinned to defeat DNS rebinding. +- Webhook URLs and the final S3 SDK hostname are SSRF-validated with the resolved IP pinned to the connection. Private, loopback, and metadata IPs are rejected. S3 signed request paths are preserved without following redirects. A configured outbound gateway that cannot preserve signed request targets fails closed. - Every create, update, delete, manual run, and test-connection call is recorded in the [Audit Log](/platform/enterprise/audit-logs). --- @@ -218,7 +220,7 @@ After data lands in your bucket or webhook system, archive lifecycle (transition }, { question: "What happens if my credentials stop working mid-run?", - answer: "The run fails, the drain cursor does not advance, and the failed run is recorded with the error. Once you fix the credentials with an Update or by re-creating the drain, the next run replays from where the last successful run left off." + answer: "The run fails and is recorded with the error. Already acknowledged chunks retain their checkpoints. Once you update the credentials, the next run resumes from the last acknowledged position; an uncheckpointed delivery may be repeated." }, { question: "What format is the data in?", @@ -264,4 +266,4 @@ curl -H "Authorization: Bearer $CRON_SECRET" \ https://sim.example.com/api/cron/run-data-drains ``` -Each due drain is then fanned out as a `run-data-drain` background job, so the deployment also needs `TRIGGER_DEV_ENABLED` and a configured Trigger.dev project. Without it the dispatcher still claims the work and enqueues it to the database, but nothing drains that queue for this job type, so the runs stay pending and never execute. See [background jobs](/platform/self-hosting/background-jobs). +Each due drain is dispatched as a `run-data-drain` background job. With Trigger.dev enabled, it runs in the configured Trigger.dev project. Otherwise the database queue executes the registered drain runner. Per-drain database claims apply in both modes. The database fallback limits active drain jobs to two per app process; Trigger.dev workers use serialized per-drain jobs. See [background jobs](/platform/self-hosting/background-jobs). diff --git a/apps/sim/app/api/audit-logs/export/route.test.ts b/apps/sim/app/api/audit-logs/export/route.test.ts deleted file mode 100644 index fe1adab3172..00000000000 --- a/apps/sim/app/api/audit-logs/export/route.test.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { authMockFns, createMockRequest } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockValidateEnterpriseAuditAccess, - mockBuildOrgScopeCondition, - mockGetOrgWorkspaceIds, - mockQueryAuditLogs, - mockBuildFilterConditions, -} = vi.hoisted(() => ({ - mockValidateEnterpriseAuditAccess: vi.fn(), - mockBuildOrgScopeCondition: vi.fn(), - mockGetOrgWorkspaceIds: vi.fn(), - mockQueryAuditLogs: vi.fn(), - mockBuildFilterConditions: vi.fn(), -})) - -vi.mock('@/app/api/v1/audit-logs/auth', () => ({ - validateEnterpriseAuditAccess: mockValidateEnterpriseAuditAccess, -})) - -vi.mock('@/lib/audit-logs/query', () => ({ - buildFilterConditions: mockBuildFilterConditions, - buildOrgScopeCondition: mockBuildOrgScopeCondition, - getOrgWorkspaceIds: mockGetOrgWorkspaceIds, - queryAuditLogs: mockQueryAuditLogs, -})) - -import { GET } from '@/app/api/audit-logs/export/route' - -const mockGetSession = authMockFns.mockGetSession - -const ORG_ID = 'org-1' -const MEMBER_IDS = ['admin-1'] -const SCOPE_SENTINEL = { type: 'org-scope-sentinel' } - -function makeRequest(query = '') { - const search = new URLSearchParams(query.startsWith('?') ? query.slice(1) : query) - search.set('organizationId', ORG_ID) - return createMockRequest( - 'GET', - undefined, - {}, - `http://localhost:3000/api/audit-logs/export?${search.toString()}` - ) -} - -function auditLog(overrides: Partial> = {}) { - return { - id: 'log-1', - workspaceId: null, - actorId: 'admin-1', - actorName: 'Ada Lovelace', - actorEmail: 'ada@example.com', - action: 'workflow.created', - resourceType: 'workflow', - resourceId: 'wf-1', - resourceName: 'My workflow', - description: null, - metadata: null, - createdAt: new Date('2026-07-01T00:00:00.000Z'), - ...overrides, - } -} - -describe('GET /api/audit-logs/export', () => { - beforeEach(() => { - mockGetSession.mockResolvedValue({ user: { id: 'admin-1' } }) - mockValidateEnterpriseAuditAccess.mockResolvedValue({ - success: true, - context: { organizationId: ORG_ID, orgMemberIds: MEMBER_IDS }, - }) - mockGetOrgWorkspaceIds.mockResolvedValue([]) - mockBuildOrgScopeCondition.mockReturnValue(SCOPE_SENTINEL) - mockBuildFilterConditions.mockReturnValue([]) - mockQueryAuditLogs.mockResolvedValue({ data: [], nextCursor: undefined }) - }) - - it('paginates through queryAuditLogs until there is no nextCursor', async () => { - mockQueryAuditLogs - .mockResolvedValueOnce({ - data: [auditLog({ id: 'log-1' })], - nextCursor: 'cursor-1', - }) - .mockResolvedValueOnce({ - data: [auditLog({ id: 'log-2' })], - nextCursor: undefined, - }) - - const response = await GET(makeRequest()) - const csv = await response.text() - - expect(mockQueryAuditLogs).toHaveBeenCalledTimes(2) - expect(mockQueryAuditLogs).toHaveBeenNthCalledWith( - 2, - expect.anything(), - expect.any(Number), - 'cursor-1' - ) - expect(csv.split('\n')).toHaveLength(3) - }) - - it('rejects an actorId that is not a current org member', async () => { - const response = await GET(makeRequest('?actorId=outsider-1')) - - expect(response.status).toBe(400) - expect(mockQueryAuditLogs).not.toHaveBeenCalled() - }) - - it('rejects a workspaceId outside the organization, as the list route does', async () => { - mockGetOrgWorkspaceIds.mockResolvedValue(['workspace-1']) - - const response = await GET(makeRequest('?workspaceId=workspace-elsewhere')) - - expect(response.status).toBe(400) - expect(mockQueryAuditLogs).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/audit-logs/export/route.ts b/apps/sim/app/api/audit-logs/export/route.ts index bcf02a37d72..d559d6c10eb 100644 --- a/apps/sim/app/api/audit-logs/export/route.ts +++ b/apps/sim/app/api/audit-logs/export/route.ts @@ -1,31 +1,13 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' import { exportAuditLogsContract } from '@/lib/api/contracts/audit-logs' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { - buildFilterConditions, - buildOrgScopeCondition, - getOrgWorkspaceIds, - queryAuditLogs, -} from '@/lib/audit-logs/query' -import { getSession } from '@/lib/auth' + defineInternalBinaryRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { exportAuditLogs } from '@/lib/audit-logs/application/export-audit-logs' +import { auditLogOperations } from '@/lib/audit-logs/application/operations' import { formatCsvValue, toCsvRow } from '@/lib/core/utils/csv' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateEnterpriseAuditAccess } from '@/app/api/v1/audit-logs/auth' -import { formatAuditLogEntry } from '@/app/api/v1/audit-logs/format' - -const logger = createLogger('AuditLogsExportAPI') - -/** - * Circuit breaker, not a UX boundary — an organization's audit trail can - * genuinely grow large over time, unlike a single user's credit ledger, so - * this is sized for "a reasonable audit review window" rather than "should - * never happen." Hitting it truncates (signaled via X-Export-Truncated), it - * doesn't error. - */ -const EXPORT_SAFETY_CAP = 10000 -const EXPORT_PAGE_SIZE = 1000 const CSV_HEADER = toCsvRow([ 'Date', @@ -34,133 +16,53 @@ const CSV_HEADER = toCsvRow([ 'Resource Name', 'Actor', 'Description', + 'Event ID', + 'Workspace ID', + 'Resource ID', + 'Actor ID', + 'IP Address', + 'User Agent', + 'Surface', + 'Metadata', ]) -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - exportAuditLogsContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { error: getValidationErrorMessage(error, 'Invalid query parameters') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - - const authResult = await validateEnterpriseAuditAccess( - session.user.id, - parsed.data.query.organizationId - ) - if (!authResult.success) { - return authResult.response - } - - const { organizationId, orgMemberIds } = authResult.context - const { actorId, workspaceId, includeDeparted } = parsed.data.query - - if (actorId && !orgMemberIds.includes(actorId)) { - return NextResponse.json( - { error: 'actorId is not a member of your organization' }, - { status: 400 } - ) - } - - const orgWorkspaceIds = await getOrgWorkspaceIds(organizationId) - /** - * The same refusal `listAuditLogs` gives. The scope predicate already makes an - * out-of-organization id return nothing, but an empty CSV and a 400 that names the - * problem are very different answers to the same bad request, and the two paths - * disagreeing about which one you get is what an audit trail cannot afford. - */ - if (workspaceId && !orgWorkspaceIds.includes(workspaceId)) { - return NextResponse.json( - { error: 'workspaceId does not belong to your organization' }, - { status: 400 } - ) - } - const scopeCondition = buildOrgScopeCondition({ - organizationId, - orgWorkspaceIds, - orgMemberIds, - includeDeparted, - }) - /** - * The whole parsed query, not a hand-listed subset. - * - * Every field of `AuditLogFilterParams` is optional, so dropping one type-checks - * silently — which is how `workspaceId` came to be accepted by the contract, - * honoured by the list route, and ignored here: an admin looking at one - * workspace's feed downloaded the entire organization's, under a truncation - * warning that blamed the date range. - */ - const filterConditions = buildFilterConditions(parsed.data.query) - const conditions = [scopeCondition, ...filterConditions] - - const rows: ReturnType[] = [] - let cursor: string | undefined - let truncated = false - while (rows.length < EXPORT_SAFETY_CAP) { - const page = await queryAuditLogs( - conditions, - Math.min(EXPORT_PAGE_SIZE, EXPORT_SAFETY_CAP - rows.length), - cursor - ) - rows.push(...page.data.map(formatAuditLogEntry)) - if (!page.nextCursor) break - truncated = rows.length >= EXPORT_SAFETY_CAP - cursor = page.nextCursor - } - - if (truncated) { - logger.warn('Audit log export truncated at safety cap', { - userId: session.user.id, - organizationId, - cap: EXPORT_SAFETY_CAP, - }) - } - - const csvLines = rows.map((log) => - toCsvRow([ - formatCsvValue(log.createdAt), - formatCsvValue(log.action), - formatCsvValue(log.resourceType), - formatCsvValue(log.resourceName), - formatCsvValue(log.actorEmail || log.actorName || 'System'), - formatCsvValue(log.description), - ]) - ) - - const csv = [CSV_HEADER, ...csvLines].join('\n') - const filename = `audit-logs-${new Date().toISOString().slice(0, 10)}.csv` - - logger.info('Exported audit logs', { - userId: session.user.id, - organizationId, - rowCount: rows.length, - }) - - return new NextResponse(csv, { - status: 200, - headers: { - 'Content-Type': 'text/csv; charset=utf-8', - 'Content-Disposition': `attachment; filename="${filename}"`, - 'Cache-Control': 'no-cache', - 'X-Export-Truncated': truncated ? '1' : '0', - }, - }) - } catch (error: unknown) { - const message = getErrorMessage(error, 'Unknown error') - logger.error('Audit logs export error', { error: message }) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } +export const GET = defineInternalBinaryRoute({ + contract: exportAuditLogsContract, + auth: internalSessionAuth, + operation: auditLogOperations.export, + rateLimit: internalRateLimits.none({ reason: 'Organization administrator CSV download' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ query }) => { + const { organizationId, includeDeparted, ...filters } = query + return { organizationId, includeDeparted, filters } + }, + useCase: exportAuditLogs, + present: ({ rows, truncated }) => ({ + body: [ + CSV_HEADER, + ...rows.map((row) => + toCsvRow( + [ + row.createdAt, + row.action, + row.resourceType, + row.resourceName, + row.actorEmail || row.actorName || 'System', + row.description, + row.id, + row.workspaceId, + row.resourceId, + row.actorId, + row.ipAddress, + row.userAgent, + row.surface, + row.metadata, + ].map((value) => formatCsvValue(value)) + ) + ), + ].join('\n'), + contentType: 'text/csv; charset=utf-8', + contentDisposition: `attachment; filename="audit-logs-${new Date().toISOString().slice(0, 10)}.csv"`, + headers: { 'Cache-Control': 'no-store', 'X-Export-Truncated': truncated ? '1' : '0' }, + }), }) diff --git a/apps/sim/app/api/auth/forget-password/route.ts b/apps/sim/app/api/auth/forget-password/route.ts index 5e022144431..75fd3f7f08a 100644 --- a/apps/sim/app/api/auth/forget-password/route.ts +++ b/apps/sim/app/api/auth/forget-password/route.ts @@ -1,8 +1,4 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { db } from '@sim/db' -import { user } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { forgetPasswordContract } from '@/lib/api/contracts' import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' @@ -69,28 +65,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { redirectTo, }, method: 'POST', + headers: request.headers, }) - const [existingUser] = await db - .select({ id: user.id, name: user.name, email: user.email }) - .from(user) - .where(eq(user.email, email)) - .limit(1) - - if (existingUser) { - recordAudit({ - actorId: existingUser.id, - actorName: existingUser.name, - actorEmail: existingUser.email, - action: AuditAction.PASSWORD_RESET_REQUESTED, - resourceType: AuditResourceType.PASSWORD, - resourceId: existingUser.id, - resourceName: existingUser.email ?? undefined, - description: `Password reset requested for ${existingUser.email}`, - request, - }) - } - return NextResponse.json({ success: true }) } catch (error) { /** diff --git a/apps/sim/app/api/auth/sso/providers/[providerId]/route.ts b/apps/sim/app/api/auth/sso/providers/[providerId]/route.ts index cf8739ddb2d..22bf9c5d96b 100644 --- a/apps/sim/app/api/auth/sso/providers/[providerId]/route.ts +++ b/apps/sim/app/api/auth/sso/providers/[providerId]/route.ts @@ -1,3 +1,4 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db, ssoProvider } from '@sim/db' import { forgetPrimaryProvider } from '@sim/db/sso-primary-provider' import { createLogger } from '@sim/logger' @@ -107,7 +108,20 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Rou } /** The organization may have just lost the provider its sign-in requirement depends on. */ - if (organizationId) invalidateSsoPolicyCache(organizationId) + if (organizationId) { + invalidateSsoPolicyCache(organizationId) + recordAudit({ + actorId: session.user.id, + actorName: session.user.name, + actorEmail: session.user.email, + action: AuditAction.ORGANIZATION_SSO_PROVIDER_DELETED, + resourceType: AuditResourceType.ORGANIZATION, + resourceId: organizationId, + description: 'Deleted organization SSO provider', + metadata: { organizationId, providerId, domain: provider.domain }, + request, + }) + } logger.info('Deleted SSO provider', { providerId, diff --git a/apps/sim/app/api/auth/sso/register/route.ts b/apps/sim/app/api/auth/sso/register/route.ts index 04c3c94c800..c0f20d11891 100644 --- a/apps/sim/app/api/auth/sso/register/route.ts +++ b/apps/sim/app/api/auth/sso/register/route.ts @@ -1,3 +1,4 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db, member, ssoDomain, ssoProvider } from '@sim/db' import { keepDomainSignInProvider, ssoProviderDomainKey } from '@sim/db/sso-primary-provider' import { createLogger } from '@sim/logger' @@ -760,6 +761,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { /** The edit may have changed whether this provider can satisfy the sign-in requirement. */ invalidateSsoPolicyCache(orgId) + recordAudit({ + actorId: session.user.id, + actorName: session.user.name, + actorEmail: session.user.email, + action: AuditAction.ORGANIZATION_SSO_PROVIDER_UPDATED, + resourceType: AuditResourceType.ORGANIZATION, + resourceId: membership.organizationId, + description: 'Updated organization SSO provider', + metadata: { + organizationId: membership.organizationId, + providerId, + providerType, + domain, + jitProvisioningEnabled, + }, + request, + }) logger.info('SSO provider updated successfully', { providerId, providerType, domain }) return NextResponse.json({ success: true, @@ -808,6 +826,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { /** A new provider can make an organization able to require single sign-on again. */ invalidateSsoPolicyCache(orgId) + recordAudit({ + actorId: session.user.id, + actorName: session.user.name, + actorEmail: session.user.email, + action: AuditAction.ORGANIZATION_SSO_PROVIDER_CREATED, + resourceType: AuditResourceType.ORGANIZATION, + resourceId: membership.organizationId, + description: 'Created organization SSO provider', + metadata: { + organizationId: membership.organizationId, + providerId: registration.providerId, + providerType, + domain, + jitProvisioningEnabled, + }, + request, + }) logger.info('SSO provider registered successfully', { providerId, providerType, diff --git a/apps/sim/background/run-data-drain.ts b/apps/sim/background/run-data-drain.ts index 9dac3ef1ac0..526aede945e 100644 --- a/apps/sim/background/run-data-drain.ts +++ b/apps/sim/background/run-data-drain.ts @@ -1,5 +1,6 @@ import { task } from '@trigger.dev/sdk' -import { runDrain } from '@/lib/data-drains/service' +import { runQueuedDrain } from '@/lib/data-drains/enqueue' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' import type { RunTrigger } from '@/lib/data-drains/types' interface RunDataDrainPayload { @@ -9,18 +10,9 @@ interface RunDataDrainPayload { export const runDataDrainTask = task({ id: 'run-data-drain', - /** - * The drain cursor commits only after the last chunk is delivered, so a run - * cut off by its ceiling restarts from the old cursor and re-exports the same - * rows. A shorter cap would loop a large drain forever. - */ - maxDuration: 5400, - /** - * Enqueue sites key runs by `data-drain:`; this limit is what makes that - * key serialize one drain's runs instead of granting each key the - * environment's full concurrency. - */ queue: { concurrencyLimit: 1 }, + maxDuration: DATA_DRAIN_LIMITS.hardDurationMs / 1000, + retry: { maxAttempts: 3 }, run: async ({ drainId, trigger }: RunDataDrainPayload, { signal }) => - runDrain(drainId, trigger, { signal }), + runQueuedDrain(drainId, trigger, signal), }) diff --git a/apps/sim/lib/api/contracts/audit-logs.ts b/apps/sim/lib/api/contracts/audit-logs.ts index df7a7dbeca4..c2d7999ec60 100644 --- a/apps/sim/lib/api/contracts/audit-logs.ts +++ b/apps/sim/lib/api/contracts/audit-logs.ts @@ -81,16 +81,13 @@ export const exportAuditLogsQuerySchema = auditLogsQuerySchema.omit({ limit: tru export type ExportAuditLogsQuery = z.output /** - * CSV download of every audit log matching the filter (no pagination). `mode: - * 'text'` because a CSV response has no JSON schema to validate; the client - * triggers this via `fetch` + blob (not `requestJson`), so there's no - * response shape for a consumer to type. + * Bounded CSV download, presented through a binary descriptor for browser downloads. */ export const exportAuditLogsContract = defineRouteContract({ method: 'GET', path: '/api/audit-logs/export', query: exportAuditLogsQuerySchema, response: { - mode: 'text', + mode: 'binary', }, }) diff --git a/apps/sim/lib/audit-logs/application/authorized-audit-log-use-case.ts b/apps/sim/lib/audit-logs/application/authorized-audit-log-use-case.ts index 740ba6fa9ec..37336ef0dc2 100644 --- a/apps/sim/lib/audit-logs/application/authorized-audit-log-use-case.ts +++ b/apps/sim/lib/audit-logs/application/authorized-audit-log-use-case.ts @@ -12,9 +12,16 @@ import { type OperationUseCase, PersonalApiKeysDisabledError, } from '@/lib/core/application' +import { + recordProjectedUseCaseAuditEntries, + type WorkspaceUseCaseAuditEntry, +} from '@/lib/core/application/authorized-workspace-use-case' import { isCopilotWorkspaceInvocation } from '@/lib/core/application/copilot-workspace-invocation' import { requireOAuthOperationScope } from '@/lib/core/application/oauth-authorization' -import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + OrchestrationError, + type OrchestrationRequestContext, +} from '@/lib/core/orchestration/types' import { refuseCapability } from '@/lib/permission-groups/capabilities' import { isCapabilityWithheldForUser } from '@/lib/permission-groups/user-scope.server' import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' @@ -33,7 +40,9 @@ interface AuthorizedAuditLogDefinition { principal: AuditLogPrincipal input: I context: AuthorizedAuditLogContext + request?: OrchestrationRequestContext }): Promise + projectAudit?(args: { input: I; result: R }): WorkspaceUseCaseAuditEntry } function requireAuditLogPrincipal( @@ -92,7 +101,7 @@ export function defineAuthorizedAuditLogUseCase input.organizationId, + execute: async ({ input, context, request }) => { + const workspaceIds = await getOrgWorkspaceIds(context.organizationId) + if (input.filters.workspaceId && !workspaceIds.includes(input.filters.workspaceId)) { + throw new OrchestrationError('validation', 'workspaceId does not belong to your organization') + } + if ( + input.filters.actorId && + !input.includeDeparted && + !context.orgMemberIds.includes(input.filters.actorId) + ) { + throw new OrchestrationError('validation', 'actorId is not a member of your organization') + } + const scope = buildOrgScopeCondition({ + organizationId: context.organizationId, + orgWorkspaceIds: workspaceIds, + orgMemberIds: context.orgMemberIds, + includeDeparted: input.includeDeparted, + }) + const conditions = [scope, ...buildFilterConditions(input.filters)] + const rows: Awaited>['data'] = [] + let cursor: string | undefined + let bytes = 0 + let truncated = false + pages: while (rows.length < EXPORT_MAX_ROWS) { + request?.signal?.throwIfAborted() + const page = await queryAuditLogs( + conditions, + Math.min(EXPORT_PAGE_ROWS, EXPORT_MAX_ROWS - rows.length), + cursor + ) + for (const row of page.data) { + bytes += Buffer.byteLength(JSON.stringify(row), 'utf8') + if (bytes > EXPORT_MAX_BYTES) { + truncated = true + break pages + } + rows.push(row) + } + if (!page.nextCursor) break + cursor = page.nextCursor + truncated = rows.length >= EXPORT_MAX_ROWS + } + return { rows, truncated } + }, + projectAudit: ({ input, result }) => ({ + action: AuditAction.AUDIT_LOGS_EXPORTED, + resourceType: AuditResourceType.ORGANIZATION, + resourceId: input.organizationId, + description: 'Exported organization audit logs', + metadata: { + format: 'csv', + rowCount: result.rows.length, + truncated: result.truncated, + includeDeparted: input.includeDeparted, + filters: Object.keys(input.filters).filter((key) => + Boolean(input.filters[key as keyof AuditLogFilterParams]) + ), + }, + }), +}) diff --git a/apps/sim/lib/audit-logs/application/operations.ts b/apps/sim/lib/audit-logs/application/operations.ts index 384d3a5527a..2f56b86ce32 100644 --- a/apps/sim/lib/audit-logs/application/operations.ts +++ b/apps/sim/lib/audit-logs/application/operations.ts @@ -15,13 +15,8 @@ export interface AuditLogOperation extends Applicati readonly authority: 'organization_admin' readonly organizationRoles: readonly ['admin', 'owner'] readonly workspaceApiKey: 'deny' - readonly oauthScope: 'api:read' - readonly principalKinds: readonly [ - 'session', - 'personal_api_key', - 'oauth_access_token', - 'delegated', - ] + readonly oauthScope?: 'api:read' + readonly principalKinds: readonly AuditLogPrincipal['kind'][] } function defineAuditLogOperation( @@ -38,6 +33,15 @@ function defineAuditLogOperation( } export const auditLogOperations = { + // permission-group-exempt: exporting the organization audit trail requires organization administrator authority + export: defineAuditLogOperation({ + id: 'audit_logs.export', + capability: 'none', + authority: 'organization_admin', + organizationRoles: ['admin', 'owner'], + workspaceApiKey: 'deny', + principalKinds: ['session'], + }), // permission-group-exempt: the organization audit trail is authorized by organization admin or owner role, a scope no workspace-shaped permission group can name list: defineAuditLogOperation({ id: 'audit_logs.list', diff --git a/apps/sim/lib/audit-logs/export.integration.ts b/apps/sim/lib/audit-logs/export.integration.ts new file mode 100644 index 00000000000..ee5e5c70328 --- /dev/null +++ b/apps/sim/lib/audit-logs/export.integration.ts @@ -0,0 +1,120 @@ +import { authMock, authMockFns } from '@sim/testing/mocks/auth.mock' +import { envFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' +import { generateId } from '@sim/utils/id' +import { NextRequest } from 'next/server' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/auth', () => authMock) +vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) + +async function loadRuntime() { + const [{ db }, schema, { eq, inArray }, { GET }] = await Promise.all([ + import('@sim/db'), + import('@sim/db/schema'), + import('drizzle-orm'), + import('@/app/api/audit-logs/export/route'), + ]) + return { db, schema, eq, inArray, GET } +} + +describe('Organization audit CSV export in PostgreSQL', () => { + let runtime: Awaited> + const organizationId = generateId() + const adminId = generateId() + const departedId = generateId() + const eventId = generateId() + + beforeAll(async () => { + runtime = await loadRuntime() + setEnvFlags({ isAuditLogsEnabled: true, isBillingEnabled: false }) + const { db, schema } = runtime + const now = new Date() + await db.insert(schema.user).values( + [adminId, departedId].map((id) => ({ + id, + name: id, + email: `${id}@audit-export.test`, + emailVerified: true, + createdAt: now, + updatedAt: now, + })) + ) + await db.insert(schema.organization).values({ + id: organizationId, + name: 'Audit export fixture', + slug: organizationId, + createdAt: now, + }) + await db.insert(schema.member).values({ + id: generateId(), + userId: adminId, + organizationId, + role: 'owner', + createdAt: now, + }) + await db.insert(schema.auditLog).values({ + id: eventId, + actorId: departedId, + action: 'organization.updated', + resourceType: 'organization', + resourceId: organizationId, + metadata: { organizationId, changedFields: ['name'] }, + ipAddress: '203.0.113.20', + userAgent: 'Audit export fixture', + surface: 'web', + }) + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: adminId }, + session: { id: generateId() }, + }) + }, 30_000) + + afterAll(async () => { + if (!runtime) return + const { db, schema, eq, inArray } = runtime + await db.delete(schema.auditLog).where(inArray(schema.auditLog.actorId, [adminId, departedId])) + await db.delete(schema.organization).where(eq(schema.organization.id, organizationId)) + await db.delete(schema.user).where(inArray(schema.user.id, [adminId, departedId])) + }) + + it('includes departed actors and forensic identifiers, and records the export', async () => { + const response = await runtime.GET( + new NextRequest( + `http://localhost:3000/api/audit-logs/export?organizationId=${organizationId}&includeDeparted=true&actorId=${departedId}` + ) + ) + expect(response.status).toBe(200) + const csv = await response.text() + expect(csv).toContain(eventId) + expect(csv).toContain(organizationId) + expect(csv).toContain('203.0.113.20') + expect(csv).toContain('changedFields') + expect(response.headers.get('cache-control')).toBe('no-store') + await vi.waitFor(async () => { + const events = await runtime.db + .select() + .from(runtime.schema.auditLog) + .where(runtime.eq(runtime.schema.auditLog.actorId, adminId)) + expect(events).toHaveLength(1) + expect(events[0].action).toBe('audit_logs.exported') + expect(events[0].metadata).toMatchObject({ organizationId, rowCount: 1, truncated: false }) + }) + }) + it('rejects workspace filters outside the authorized organization', async () => { + const response = await runtime.GET( + new NextRequest( + `http://localhost:3000/api/audit-logs/export?organizationId=${organizationId}&workspaceId=${generateId()}` + ) + ) + expect(response.status).toBe(400) + }) + + it('rejects a departed actor unless historical access was requested', async () => { + const response = await runtime.GET( + new NextRequest( + `http://localhost:3000/api/audit-logs/export?organizationId=${organizationId}&actorId=${departedId}` + ) + ) + expect(response.status).toBe(400) + }) +}) diff --git a/apps/sim/lib/audit-logs/query.integration.ts b/apps/sim/lib/audit-logs/query.integration.ts new file mode 100644 index 00000000000..4f24933206c --- /dev/null +++ b/apps/sim/lib/audit-logs/query.integration.ts @@ -0,0 +1,81 @@ +import { generateId } from '@sim/utils/id' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +async function loadRuntime() { + const [{ db }, { auditLog }, { eq, inArray, sql }, query, { auditLogsSource }] = + await Promise.all([ + import('@sim/db'), + import('@sim/db/schema'), + import('drizzle-orm'), + import('@/lib/audit-logs/query'), + import('@/lib/data-drains/sources/audit-logs'), + ]) + return { db, auditLog, eq, inArray, sql, query, auditLogsSource } +} + +describe('Audit pagination and drain tenant coverage in PostgreSQL', () => { + let runtime: Awaited> + const prefix = generateId() + const organizationId = generateId() + const ids = ['a', 'b', 'c', 'other'].map((suffix) => `${prefix}-${suffix}`) + + beforeAll(async () => { + runtime = await loadRuntime() + await runtime.db.insert(runtime.auditLog).values( + ids.map((id, index) => ({ + id, + actorId: null, + action: 'organization.updated', + resourceType: 'organization', + resourceId: index === 3 ? generateId() : organizationId, + metadata: index === 0 ? { organizationId } : {}, + createdAt: new Date('2026-01-01T12:00:00.123Z'), + })) + ) + for (const [index, id] of ids.entries()) { + await runtime.db + .update(runtime.auditLog) + .set({ + createdAt: runtime.sql`timestamp '2026-01-01 12:00:00.123' + ${index + 1} * interval '100 microseconds'`, + }) + .where(runtime.eq(runtime.auditLog.id, id)) + } + }, 30_000) + + afterAll(async () => { + if (runtime) { + await runtime.db.delete(runtime.auditLog).where(runtime.inArray(runtime.auditLog.id, ids)) + } + }) + + it('returns every event once when pages split within a PostgreSQL millisecond', async () => { + const seen: string[] = [] + let cursor: string | undefined + for (let pageIndex = 0; pageIndex < ids.length + 1; pageIndex++) { + const page = await runtime.query.queryAuditLogs( + [runtime.inArray(runtime.auditLog.id, ids)], + 1, + cursor + ) + seen.push(...page.data.map((row) => row.id)) + if (!page.nextCursor) break + cursor = page.nextCursor + } + expect(seen).toHaveLength(ids.length) + expect(new Set(seen)).toEqual(new Set(ids)) + }) + + it('drains all organization-resource events and excludes the other tenant', async () => { + const seen: string[] = [] + for await (const page of runtime.auditLogsSource.pages({ + organizationId, + cursor: null, + chunkSize: 1, + signal: new AbortController().signal, + })) { + seen.push(...page.map((row) => row.id)) + } + expect(new Set(seen)).toEqual(new Set(ids.slice(0, 3))) + expect(seen).toHaveLength(3) + }) +}) diff --git a/apps/sim/lib/audit-logs/query.ts b/apps/sim/lib/audit-logs/query.ts index 9c84d67751a..a31bf0ff667 100644 --- a/apps/sim/lib/audit-logs/query.ts +++ b/apps/sim/lib/audit-logs/query.ts @@ -100,7 +100,7 @@ export async function getOrgWorkspaceIds(organizationId: string): Promise { ) )! - const orgScope = - orgWorkspaceIds.length > 0 - ? or(inArray(auditLog.workspaceId, orgWorkspaceIds), orgLevelCondition)! - : orgLevelCondition + const workspaceCondition = + orgWorkspaceIds === undefined + ? sql`exists (select 1 from ${workspace} where ${workspace.id} = ${auditLog.workspaceId} and ${workspace.organizationId} = ${organizationId})` + : orgWorkspaceIds.length > 0 + ? inArray(auditLog.workspaceId, orgWorkspaceIds) + : undefined + const orgScope = workspaceCondition + ? or(workspaceCondition, orgLevelCondition)! + : orgLevelCondition if (includeDeparted) return orgScope @@ -139,10 +144,14 @@ function buildCursorCondition(cursor: string): SQL | null { const cursorData = decodeAuditLogCursor(cursor) if (!cursorData) return null const cursorDate = new Date(cursorData.createdAt) + const cursorTimestamp = sql`date_trunc('milliseconds', ${auditLog.createdAt})` return or( - lt(auditLog.createdAt, cursorDate), - and(eq(auditLog.createdAt, cursorDate), lt(auditLog.id, cursorData.id)) + lt(cursorTimestamp, sql.param(cursorDate, auditLog.createdAt)), + and( + eq(cursorTimestamp, sql.param(cursorDate, auditLog.createdAt)), + lt(auditLog.id, cursorData.id) + ) )! } @@ -166,7 +175,7 @@ export async function queryAuditLogs( .select() .from(auditLog) .where(allConditions.length > 0 ? and(...allConditions) : undefined) - .orderBy(desc(auditLog.createdAt), desc(auditLog.id)) + .orderBy(desc(sql`date_trunc('milliseconds', ${auditLog.createdAt})`), desc(auditLog.id)) .limit(limit + 1) const hasMore = rows.length > limit diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index 4b317187d5a..b6ce68fa447 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -46,6 +46,13 @@ import { isSignInProviderAllowed, } from '@/lib/auth/constants' import { getAuthDatabase } from '@/lib/auth/database-context' +import { + type AuthenticationSecurityAuditSnapshot, + captureAuthenticationSecurityAudit, + recordAuthenticationAudit, + recordAuthenticationSecurityAudit, + recordPasswordResetAudit, +} from '@/lib/auth/lifecycle-audit' import { hashOAuthToken } from '@/lib/auth/oauth-access-token' import { consentRequestNamesClient, @@ -904,208 +911,259 @@ export const auth = betterAuth({ throw new Error(`Failed to send reset password email: ${result.message}`) } }, - onPasswordReset: async ({ user: resetUser }) => { - const { AuditAction, AuditResourceType, recordAudit } = await import('@sim/audit') - recordAudit({ - actorId: resetUser.id, - actorName: resetUser.name, - actorEmail: resetUser.email, - action: AuditAction.PASSWORD_RESET, - resourceType: AuditResourceType.PASSWORD, - resourceId: resetUser.id, - description: `Password reset completed for ${resetUser.email}`, - }) + onPasswordReset: async ({ user: resetUser }, request) => { + await recordPasswordResetAudit({ kind: 'completed', userId: resetUser.id, request }) }, }, hooks: { before: createAuthMiddleware(async (ctx) => { - /** Refuse provider calls when user authentication is disabled without blocking connector OAuth. */ - if ( - ((ctx.path.startsWith('/oauth2/') && - ctx.path !== '/oauth2/link' && - !ctx.path.startsWith('/oauth2/callback/')) || - ctx.path === '/.well-known/oauth-authorization-server') && - isAuthDisabled - ) { - throw new APIError('NOT_FOUND', { message: 'OAuth provider is not enabled' }) - } - - /** - * Better Auth 1.6.27 re-enters OAuth authorization when its own session - * refresh sets a cookie, issuing a second code that is never returned. - * Suppressing sliding renewal only for this request prevents the orphan; - * the next ordinary session request can still renew the same session. - */ - if (ctx.path === '/oauth2/authorize') await setShouldSkipSessionRefresh(true) - - /** - * Restrict the unauthenticated sign-in endpoints to first-party login - * providers. Better Auth registers every generic-OAuth integration - * connector as a social provider, so without this guard `microsoft-ad`, - * `salesforce`, `jira`, and the rest are reachable through - * `/sign-in/social` and `/sign-in/oauth2` and can mint a session for any - * user by email (nOAuth account takeover). Connectors are connected only - * through the authenticated `/oauth2/link` flow, which is unaffected. - */ - if (ctx.path === '/sign-in/social' || ctx.path === '/sign-in/oauth2') { - const requestedProviderId = getRequestedSignInProviderId(ctx.path, ctx.body) - if (!isSignInProviderAllowed(requestedProviderId)) { - throw new APIError('FORBIDDEN', { - message: - 'This provider can only be connected from a signed-in account and cannot be used to sign in.', - }) + try { + /** Refuse provider calls when user authentication is disabled without blocking connector OAuth. */ + if ( + ((ctx.path.startsWith('/oauth2/') && + ctx.path !== '/oauth2/link' && + !ctx.path.startsWith('/oauth2/callback/')) || + ctx.path === '/.well-known/oauth-authorization-server') && + isAuthDisabled + ) { + throw new APIError('NOT_FOUND', { message: 'OAuth provider is not enabled' }) } - } - /** - * permission-group-enforced: oauth_apps.use, cli.use — account-level - * authorization uses the default group; token issuance rechecks it later. - * Explicit denial remains available even when access has been withheld. - */ - if ( - ctx.path === '/oauth2/authorize' || - (ctx.path === '/oauth2/consent' && ctx.body?.accept === true) - ) { - const session = await getSessionFromCtx(ctx) - const userId = session?.user?.id - if (userId) { - if (await isCapabilityWithheldForUser(userId, 'oauth_apps.use')) { - throw new APIError('FORBIDDEN', { - message: capabilityRefusal('oauth_apps.use'), - error: 'access_denied', - error_description: capabilityRefusal('oauth_apps.use'), - }) - } - const isCli = - ctx.path === '/oauth2/authorize' - ? ctx.query?.client_id === SIM_CLI_CLIENT_ID - : consentRequestNamesClient(ctx.body?.oauth_query, SIM_CLI_CLIENT_ID) - if (isCli && (await isCapabilityWithheldForUser(userId, 'cli.use'))) { + /** + * Better Auth 1.6.27 re-enters OAuth authorization when its own session + * refresh sets a cookie, issuing a second code that is never returned. + * Suppressing sliding renewal only for this request prevents the orphan; + * the next ordinary session request can still renew the same session. + */ + if (ctx.path === '/oauth2/authorize') await setShouldSkipSessionRefresh(true) + + /** + * Restrict the unauthenticated sign-in endpoints to first-party login + * providers. Better Auth registers every generic-OAuth integration + * connector as a social provider, so without this guard `microsoft-ad`, + * `salesforce`, `jira`, and the rest are reachable through + * `/sign-in/social` and `/sign-in/oauth2` and can mint a session for any + * user by email (nOAuth account takeover). Connectors are connected only + * through the authenticated `/oauth2/link` flow, which is unaffected. + */ + if (ctx.path === '/sign-in/social' || ctx.path === '/sign-in/oauth2') { + const requestedProviderId = getRequestedSignInProviderId(ctx.path, ctx.body) + if (!isSignInProviderAllowed(requestedProviderId)) { throw new APIError('FORBIDDEN', { - message: capabilityRefusal('cli.use'), - error: 'access_denied', - error_description: capabilityRefusal('cli.use'), + message: + 'This provider can only be connected from a signed-in account and cannot be used to sign in.', }) } } - } - if (ctx.path === '/oauth2/link' && ctx.body?.providerId === MICROSOFT_DATAVERSE_PROVIDER_ID) { - try { - assertMicrosoftDataverseOAuthLinkRequest( - ctx.body.callbackURL, - ctx.body.scopes, - getCanonicalScopesForProvider(MICROSOFT_DATAVERSE_PROVIDER_ID) - ) - } catch (error) { - throw new APIError('BAD_REQUEST', { - message: getErrorMessage(error, 'Invalid Dataverse OAuth request'), - }) + /** + * permission-group-enforced: oauth_apps.use, cli.use — account-level + * authorization uses the default group; token issuance rechecks it later. + * Explicit denial remains available even when access has been withheld. + */ + if ( + ctx.path === '/oauth2/authorize' || + (ctx.path === '/oauth2/consent' && ctx.body?.accept === true) + ) { + const session = await getSessionFromCtx(ctx) + const userId = session?.user?.id + if (userId) { + if (await isCapabilityWithheldForUser(userId, 'oauth_apps.use')) { + throw new APIError('FORBIDDEN', { + message: capabilityRefusal('oauth_apps.use'), + error: 'access_denied', + error_description: capabilityRefusal('oauth_apps.use'), + }) + } + const isCli = + ctx.path === '/oauth2/authorize' + ? ctx.query?.client_id === SIM_CLI_CLIENT_ID + : consentRequestNamesClient(ctx.body?.oauth_query, SIM_CLI_CLIENT_ID) + if (isCli && (await isCapabilityWithheldForUser(userId, 'cli.use'))) { + throw new APIError('FORBIDDEN', { + message: capabilityRefusal('cli.use'), + error: 'access_denied', + error_description: capabilityRefusal('cli.use'), + }) + } + } } - } - if (ctx.path.startsWith('/sign-up') && isRegistrationDisabled) - throw new APIError('FORBIDDEN', { - message: 'Registration is disabled, please contact your admin.', - }) + if ( + ctx.path === '/oauth2/link' && + ctx.body?.providerId === MICROSOFT_DATAVERSE_PROVIDER_ID + ) { + try { + assertMicrosoftDataverseOAuthLinkRequest( + ctx.body.callbackURL, + ctx.body.scopes, + getCanonicalScopesForProvider(MICROSOFT_DATAVERSE_PROVIDER_ID) + ) + } catch (error) { + throw new APIError('BAD_REQUEST', { + message: getErrorMessage(error, 'Invalid Dataverse OAuth request'), + }) + } + } - if (!isEmailPasswordEnabled) { - const emailPasswordPaths = ['/sign-in/email', '/sign-up/email', '/email-otp'] - if (emailPasswordPaths.some((path) => ctx.path.startsWith(path))) + if (ctx.path.startsWith('/sign-up') && isRegistrationDisabled) throw new APIError('FORBIDDEN', { - message: 'Email/password authentication is disabled. Please use SSO to sign in.', + message: 'Registration is disabled, please contact your admin.', }) - } - const isSignIn = ctx.path.startsWith('/sign-in') - const isSignUp = ctx.path.startsWith('/sign-up') - - if (isSignIn || isSignUp) { - const accessControl = await getAccessControlConfig() - const requestEmail = ctx.body?.email?.toLowerCase() - - // Banning an existing account is owned by better-auth's admin plugin (a - // `session.create.before` hook that blocks banned users at sign-in across - // all providers), so it is not re-checked here. - const hasAllowlist = - accessControl.allowedLoginEmails.length > 0 || - accessControl.allowedLoginDomains.length > 0 - if (hasAllowlist && requestEmail) { - const emailDomain = requestEmail.split('@')[1] - const isAllowed = - accessControl.allowedLoginEmails.includes(requestEmail) || - (!!emailDomain && accessControl.allowedLoginDomains.includes(emailDomain)) - if (!isAllowed) { + if (!isEmailPasswordEnabled) { + const emailPasswordPaths = ['/sign-in/email', '/sign-up/email', '/email-otp'] + if (emailPasswordPaths.some((path) => ctx.path.startsWith(path))) throw new APIError('FORBIDDEN', { - message: 'Access restricted. Please contact your administrator.', + message: 'Email/password authentication is disabled. Please use SSO to sign in.', }) - } } - // Blocked emails/domains gate both signup and sign-in. OAuth/SSO sign-ins - // have no email in the body here; the session.create.before hook covers them. - if (isEmailBlockedByAccessControl(requestEmail, accessControl)) { - throw new APIError('FORBIDDEN', { - message: isSignUp - ? 'Sign-ups from this email are not allowed.' - : 'Access restricted. Please contact your administrator.', - }) - } + const isSignIn = ctx.path.startsWith('/sign-in') + const isSignUp = ctx.path.startsWith('/sign-up') - if ( - isSignupMxValidationEnabled && - ctx.path.startsWith('/sign-up/email') && - ctx.body?.email - ) { - const mxCheck = await validateSignupEmailMx( - ctx.body.email, - accessControl.blockedEmailMxHosts - ) - if (!mxCheck.allowed) { + if (isSignIn || isSignUp) { + const accessControl = await getAccessControlConfig() + const requestEmail = ctx.body?.email?.toLowerCase() + + // Banning an existing account is owned by better-auth's admin plugin (a + // `session.create.before` hook that blocks banned users at sign-in across + // all providers), so it is not re-checked here. + const hasAllowlist = + accessControl.allowedLoginEmails.length > 0 || + accessControl.allowedLoginDomains.length > 0 + if (hasAllowlist && requestEmail) { + const emailDomain = requestEmail.split('@')[1] + const isAllowed = + accessControl.allowedLoginEmails.includes(requestEmail) || + (!!emailDomain && accessControl.allowedLoginDomains.includes(emailDomain)) + if (!isAllowed) { + throw new APIError('FORBIDDEN', { + message: 'Access restricted. Please contact your administrator.', + }) + } + } + + // Blocked emails/domains gate both signup and sign-in. OAuth/SSO sign-ins + // have no email in the body here; the session.create.before hook covers them. + if (isEmailBlockedByAccessControl(requestEmail, accessControl)) { throw new APIError('FORBIDDEN', { - message: 'Sign-ups from this email domain are not allowed.', + message: isSignUp + ? 'Sign-ups from this email are not allowed.' + : 'Access restricted. Please contact your administrator.', }) } + + if ( + isSignupMxValidationEnabled && + ctx.path.startsWith('/sign-up/email') && + ctx.body?.email + ) { + const mxCheck = await validateSignupEmailMx( + ctx.body.email, + accessControl.blockedEmailMxHosts + ) + if (!mxCheck.allowed) { + throw new APIError('FORBIDDEN', { + message: 'Sign-ups from this email domain are not allowed.', + }) + } + } } - } - /** - * Personal checkout guard. The Stripe plugin's `authorizeReference` - * only runs for organization references (it skips references equal to - * the session user), so personal checkout admission lives here. It - * prevents both a duplicate checkout while Stripe payment is pending - * and a personal plan for someone already covered by an organization. - */ - if (isBillingEnabled && ctx.path === '/subscription/upgrade') { - const session = await getSessionFromCtx(ctx) - const sessionUserId = session?.user?.id - if (sessionUserId) { - const requestBody = ctx.body ?? {} - const referenceId = resolveCheckoutReferenceId( - requestBody, - sessionUserId, - getActiveOrganizationId(session) - ) - if (referenceId) { - const checkoutAdmissionClaim = await claimCheckoutAdmission(referenceId) - try { - if (isPersonalCheckoutRequest(requestBody, sessionUserId)) { - await assertPersonalCheckoutAllowed(sessionUserId) + /** + * Personal checkout guard. The Stripe plugin's `authorizeReference` + * only runs for organization references (it skips references equal to + * the session user), so personal checkout admission lives here. It + * prevents both a duplicate checkout while Stripe payment is pending + * and a personal plan for someone already covered by an organization. + */ + if (isBillingEnabled && ctx.path === '/subscription/upgrade') { + const session = await getSessionFromCtx(ctx) + const sessionUserId = session?.user?.id + if (sessionUserId) { + const requestBody = ctx.body ?? {} + const referenceId = resolveCheckoutReferenceId( + requestBody, + sessionUserId, + getActiveOrganizationId(session) + ) + if (referenceId) { + const checkoutAdmissionClaim = await claimCheckoutAdmission(referenceId) + try { + if (isPersonalCheckoutRequest(requestBody, sessionUserId)) { + await assertPersonalCheckoutAllowed(sessionUserId) + } + } catch (error) { + await releaseCheckoutAdmission(checkoutAdmissionClaim) + throw error } - } catch (error) { - await releaseCheckoutAdmission(checkoutAdmissionClaim) - throw error + return { context: { billingCheckoutAdmissionClaim: checkoutAdmissionClaim } } } - return { context: { billingCheckoutAdmissionClaim: checkoutAdmissionClaim } } } } - } - return + if (ctx.path === '/sign-out') { + const session = await getSessionFromCtx(ctx) + return { context: { authenticationAuditLogoutUserId: session?.user?.id } } + } + const securityAudit = await captureAuthenticationSecurityAudit({ + path: ctx.path, + resolveActor: async () => { + const session = await getSessionFromCtx(ctx) + if (!session?.user || !session.session) return null + return { + userId: session.user.id, + sessionId: session.session.id, + role: session.user.role, + impersonatedBy: session.session.impersonatedBy, + } + }, + requestedUserId: ctx.body?.userId, + requestedToken: + ctx.path === '/admin/revoke-user-session' ? ctx.body?.sessionToken : ctx.body?.token, + }) + if (securityAudit) return { context: { authenticationSecurityAudit: securityAudit } } + return + } catch (error) { + await recordAuthenticationAudit({ + path: ctx.path, + email: ctx.body?.email, + providerId: ctx.params?.providerId, + returned: error, + request: ctx.request ?? (ctx.headers ? { headers: ctx.headers } : undefined), + }) + throw error + } }), after: createAuthMiddleware(async (ctx) => { if (isBillingEnabled) await recordCustomerRestoreAfterHook(ctx) + const recordFinalAuthenticationResult = async ( + admittedSsoProviderId?: string, + ssoAdmissionCompleted = false + ) => { + const auditContext = ctx as typeof ctx & { + authenticationAuditLogoutUserId?: string + authenticationSecurityAudit?: AuthenticationSecurityAuditSnapshot + } + await recordAuthenticationSecurityAudit({ + snapshot: auditContext.authenticationSecurityAudit, + returned: ctx.context.returned, + request: ctx.request ?? (ctx.headers ? { headers: ctx.headers } : undefined), + }) + return recordAuthenticationAudit({ + path: ctx.path, + userId: ctx.context.newSession?.user?.id, + email: ctx.body?.email, + logoutUserId: auditContext.authenticationAuditLogoutUserId, + providerId: admittedSsoProviderId ?? ctx.params?.providerId ?? ctx.params?.id, + ssoAdmissionCompleted, + returned: ctx.context.returned, + redirectLocation: ctx.context.responseHeaders?.get('location'), + request: ctx.request ?? (ctx.headers ? { headers: ctx.headers } : undefined), + }) + } if (isBillingEnabled && ctx.path === '/subscription/upgrade') { const checkoutContext = ctx as typeof ctx & { billingCheckoutAdmissionClaim?: CheckoutAdmissionClaim @@ -1115,17 +1173,26 @@ export const auth = betterAuth({ } } - if (!isSsoEnabled) return + if (!isSsoEnabled) { + await recordFinalAuthenticationResult() + return + } const oauthState = ctx.path === '/sso/callback' ? await getOAuthState() : null const providerId = resolveSsoCallbackProviderId({ path: ctx.path, routeProviderId: ctx.params?.providerId, stateProviderId: oauthState?.ssoProviderId, }) - if (!providerId) return + if (!providerId) { + await recordFinalAuthenticationResult() + return + } const newSession = ctx.context.newSession - if (!newSession?.session || !newSession.user) return + if (!newSession?.session || !newSession.user) { + await recordFinalAuthenticationResult(providerId) + return + } let admissionErrorCode: string | null = null try { @@ -1186,7 +1253,18 @@ export const auth = betterAuth({ }) } - if (!admissionErrorCode) return + if (!admissionErrorCode) { + await recordFinalAuthenticationResult(providerId, true) + return + } + + await recordAuthenticationAudit({ + path: ctx.path, + userId: newSession.user.id, + providerId, + failureReason: 'sso_admission_denied', + request: ctx.request ?? (ctx.headers ? { headers: ctx.headers } : undefined), + }) try { await ctx.context.internalAdapter.deleteSession(newSession.session.token) diff --git a/apps/sim/lib/auth/lifecycle-audit.integration.ts b/apps/sim/lib/auth/lifecycle-audit.integration.ts new file mode 100644 index 00000000000..1ec204eeeae --- /dev/null +++ b/apps/sim/lib/auth/lifecycle-audit.integration.ts @@ -0,0 +1,387 @@ +import { db } from '@sim/db' +import { + account, + auditLog, + member, + organization, + session, + ssoProvider, + user, + verification, +} from '@sim/db/schema' +import { emailMailerMock, emailMailerMockFns } from '@sim/testing/mocks/email-mailer.mock' +import { emailTemplatesMock } from '@sim/testing/mocks/email-templates.mock' +import { posthogServerMock } from '@sim/testing/mocks/posthog-server.mock' +import { createMockRequest } from '@sim/testing/mocks/request.mock' +import { generateId } from '@sim/utils/id' +import { APIError } from 'better-auth/api' +import { hashPassword } from 'better-auth/crypto' +import { count, eq, inArray, or } from 'drizzle-orm' +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { recordAuthenticationAudit } from '@/lib/auth/lifecycle-audit' + +vi.mock('@/lib/messaging/email/mailer', () => emailMailerMock) +vi.mock('@/components/emails', () => emailTemplatesMock) +vi.mock('@/lib/posthog/server', () => posthogServerMock) + +describe('authentication audit history in PostgreSQL', () => { + let auth: typeof import('@/lib/auth/auth').auth + let resetPassword: typeof import('@/app/api/auth/forget-password/route').POST + let userId: string + let organizationId: string + let email: string + let passwordHash: string + let additionalUserIds: string[] + const fixturePassword = 'Audit-fixture-password-only-36!' + + beforeAll(async () => { + ;({ auth } = await import('@/lib/auth/auth')) + ;({ POST: resetPassword } = await import('@/app/api/auth/forget-password/route')) + passwordHash = await hashPassword(fixturePassword) + }, 90_000) + + beforeEach(async () => { + userId = generateId() + organizationId = generateId() + email = `${userId}@authentication-audit.test` + additionalUserIds = [] + const now = new Date() + emailMailerMockFns.mockSendEmail.mockResolvedValue({ success: true }) + await db.insert(user).values({ + id: userId, + name: 'Authentication audit fixture', + email, + emailVerified: true, + createdAt: now, + updatedAt: now, + }) + await db.insert(organization).values({ + id: organizationId, + name: 'Authentication audit fixture', + slug: organizationId, + createdAt: now, + }) + await db.insert(member).values({ id: generateId(), userId, organizationId, role: 'member' }) + await db.insert(account).values({ + id: generateId(), + accountId: userId, + userId, + providerId: 'credential', + password: passwordHash, + createdAt: now, + updatedAt: now, + }) + }) + + afterEach(async () => { + await db.delete(auditLog).where(inArray(auditLog.resourceId, [userId, ...additionalUserIds])) + await db + .delete(verification) + .where( + or(eq(verification.value, userId), eq(verification.identifier, `sign-in-otp-${email}`)) + ) + await db.delete(organization).where(eq(organization.id, organizationId)) + await db.delete(user).where(inArray(user.id, [userId, ...additionalUserIds])) + }) + + function request(path: string, body: object, cookie?: string) { + return new Request(`http://localhost:3000/api/auth${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + origin: 'http://localhost:3000', + 'user-agent': 'Authentication audit fixture', + ...(cookie ? { cookie } : {}), + }, + body: JSON.stringify(body), + }) + } + + async function history() { + return db.select().from(auditLog).where(eq(auditLog.resourceId, userId)) + } + + async function expectEvent(action: string, actorId: string | null) { + await expect + .poll(async () => (await history()).filter((entry) => entry.action === action).length) + .toBe(1) + const entry = (await history()).find((row) => row.action === action) + expect(entry).toMatchObject({ + actorId, + resourceId: userId, + metadata: { organizationId }, + userAgent: 'Authentication audit fixture', + }) + expect(JSON.stringify(entry)).not.toContain(fixturePassword) + expect(JSON.stringify(entry)).not.toContain(passwordHash) + return entry + } + + function cookieHeader(response: Response, previous = '') { + const cookies = new Map() + for (const value of [...previous.split('; '), ...response.headers.getSetCookie()]) { + const pair = value.split(';')[0] + const separator = pair.indexOf('=') + if (separator > 0) cookies.set(pair.slice(0, separator), pair.slice(separator + 1)) + } + return [...cookies].map(([name, value]) => `${name}=${value}`).join('; ') + } + + async function login(loginEmail = email) { + const response = await auth.handler( + request('/sign-in/email', { email: loginEmail, password: fixturePassword }) + ) + expect(response.status).toBe(200) + const payload = (await response.json()) as { token: string } + return { cookie: cookieHeader(response), token: payload.token } + } + + async function createAdministrator() { + const id = generateId() + additionalUserIds.push(id) + const administratorEmail = `${id}@authentication-audit.test` + const now = new Date() + await db.insert(user).values({ + id, + name: 'Authentication audit administrator', + email: administratorEmail, + emailVerified: true, + role: 'admin', + createdAt: now, + updatedAt: now, + }) + await db.insert(account).values({ + id: generateId(), + accountId: id, + userId: id, + providerId: 'credential', + password: passwordHash, + createdAt: now, + updatedAt: now, + }) + return { id, ...(await login(administratorEmail)) } + } + + async function sessionCount() { + const [row] = await db + .select({ total: count() }) + .from(session) + .where(eq(session.userId, userId)) + return row.total + } + + it('records accepted login and logout without storing a session bearer token', async () => { + const login = await auth.handler( + request('/sign-in/email', { email, password: fixturePassword }) + ) + expect(login.status).toBe(200) + const payload = (await login.json()) as { token: string } + await expectEvent('auth.login_succeeded', userId) + const cookie = login.headers + .getSetCookie() + .map((value) => value.split(';')[0]) + .join('; ') + const logout = await auth.handler(request('/sign-out', {}, cookie)) + expect(logout.status).toBe(200) + await expectEvent('auth.logout', userId) + expect(JSON.stringify(await history())).not.toContain(payload.token) + }) + + it('records an invalid password as an unauthenticated attempt against the account', async () => { + const response = await auth.handler(request('/sign-in/email', { email, password: 'incorrect' })) + expect(response.status).toBe(401) + await expectEvent('auth.login_failed', null) + expect((await history()).some((entry) => entry.action === 'auth.login_succeeded')).toBe(false) + }) + + it('records suspended-account denial without claiming a successful login', async () => { + await db.update(user).set({ suspendedAt: new Date() }).where(eq(user.id, userId)) + const response = await auth.handler( + request('/sign-in/email', { email, password: fixturePassword }) + ) + expect(response.status).toBe(403) + await expectEvent('auth.login_failed', null) + expect((await history()).some((entry) => entry.action === 'auth.login_succeeded')).toBe(false) + }) + + it('records an accepted email OTP login without retaining the OTP or session token', async () => { + const otp = await auth.api.createVerificationOTP({ body: { email, type: 'sign-in' } }) + const response = await auth.handler(request('/sign-in/email-otp', { email, otp })) + expect(response.status).toBe(200) + const payload = (await response.json()) as { token: string } + const entry = await expectEvent('auth.login_succeeded', userId) + expect(entry?.metadata).toMatchObject({ authenticationMethod: 'email_otp' }) + expect(JSON.stringify(entry)).not.toContain(otp) + expect(JSON.stringify(entry)).not.toContain(payload.token) + }) + + it('records a rejected email OTP as an anonymous attempt against the account', async () => { + await auth.api.createVerificationOTP({ body: { email, type: 'sign-in' } }) + const response = await auth.handler(request('/sign-in/email-otp', { email, otp: 'incorrect' })) + expect(response.status).toBe(400) + const entry = await expectEvent('auth.login_failed', null) + expect(entry?.metadata).toMatchObject({ authenticationMethod: 'email_otp' }) + expect((await history()).some((event) => event.action === 'auth.login_succeeded')).toBe(false) + }) + + it('attributes a password-reset request to its unauthenticated requester and the target organization', async () => { + const response = await resetPassword( + createMockRequest({ + method: 'POST', + url: 'http://localhost:3000/api/auth/forget-password', + body: { email, redirectTo: 'http://localhost:3000/reset-password' }, + headers: { 'user-agent': 'Authentication audit fixture' }, + }), + {} + ) + expect(response.status).toBe(200) + await expectEvent('password.reset_requested', null) + }) + + it('records a reset delegated directly to the auth handler with the same anonymous actor and organization', async () => { + const response = await auth.handler( + request('/request-password-reset', { + email, + redirectTo: 'http://localhost:3000/reset-password', + }) + ) + expect(response.status).toBe(200) + await expectEvent('password.reset_requested', null) + }) + + it('does not report a rejected direct reset request as a successful reset request', async () => { + const response = await auth.handler( + request('/request-password-reset', { + email, + redirectTo: 123, + }) + ) + expect(response.status).toBe(400) + const login = await auth.handler( + request('/sign-in/email', { email, password: fixturePassword }) + ) + expect(login.status).toBe(200) + await expectEvent('auth.login_succeeded', userId) + expect( + (await history()).filter((entry) => entry.action === 'password.reset_requested') + ).toHaveLength(0) + }) + + it('scopes an SSO admission refusal to the configured provider organization when the account belongs elsewhere', async () => { + const providerOrganizationId = generateId() + const providerId = generateId() + await db.insert(organization).values({ + id: providerOrganizationId, + name: 'SSO authentication audit fixture', + slug: providerOrganizationId, + createdAt: new Date(), + }) + await db.insert(ssoProvider).values({ + id: generateId(), + issuer: 'https://sso.authentication-audit.test', + domain: 'authentication-audit.test', + userId, + providerId, + organizationId: providerOrganizationId, + }) + try { + await recordAuthenticationAudit({ + path: '/sso/callback/:providerId', + userId, + providerId, + failureReason: 'sso_admission_denied', + returned: new APIError('FORBIDDEN', { message: fixturePassword }), + request: request('/sso/callback', {}), + }) + await expect.poll(async () => (await history()).length).toBe(1) + const [entry] = await history() + expect(entry).toMatchObject({ + action: 'auth.login_failed', + actorId: null, + resourceId: userId, + metadata: { + organizationId: providerOrganizationId, + providerId, + failureReason: 'sso_admission_denied', + }, + }) + expect(JSON.stringify(entry)).not.toContain(fixturePassword) + } finally { + await db.delete(organization).where(eq(organization.id, providerOrganizationId)) + } + }) + it.each(['started', 'ended'] as const)( + 'records impersonation %s against the target organization with the administrator as actor', + async (outcome) => { + const administrator = await createAdministrator() + const started = await auth.handler( + request('/admin/impersonate-user', { userId }, administrator.cookie) + ) + expect(started.status).toBe(200) + const payload = (await started.json()) as { session: { token: string } } + const ended = await auth.handler( + request('/admin/stop-impersonating', {}, cookieHeader(started, administrator.cookie)) + ) + expect(ended.status).toBe(200) + await expectEvent(`auth.impersonation_${outcome}`, administrator.id) + const entries = JSON.stringify(await history()) + expect(entries).not.toContain(administrator.token) + expect(entries).not.toContain(payload.session.token) + } + ) + + it.each([ + { path: '/revoke-session', mode: 'single', matched: 1, remaining: 1 }, + { path: '/revoke-sessions', mode: 'all', matched: 2, remaining: 0 }, + { path: '/revoke-other-sessions', mode: 'other', matched: 1, remaining: 1 }, + ] as const)( + 'records $mode self session revocation without retaining the session bearer tokens', + async ({ path, mode, matched, remaining }) => { + const previous = await login() + const current = await login() + const response = await auth.handler( + request(path, mode === 'single' ? { token: previous.token } : {}, current.cookie) + ) + expect(response.status).toBe(200) + expect(await sessionCount()).toBe(remaining) + const entry = await expectEvent('auth.sessions_revoked', userId) + expect(entry?.metadata).toMatchObject({ + revocationMode: mode, + sessionsMatchedBefore: matched, + }) + const entries = JSON.stringify(await history()) + expect(entries).not.toContain(previous.token) + expect(entries).not.toContain(current.token) + } + ) + + it.each([ + { path: '/admin/revoke-user-session', mode: 'single', matched: 1, remaining: 1 }, + { path: '/admin/revoke-user-sessions', mode: 'all', matched: 2, remaining: 0 }, + ] as const)( + 'records $mode administrative session revocation in the target organization with the real operator', + async ({ path, mode, matched, remaining }) => { + const previous = await login() + const current = await login() + const administrator = await createAdministrator() + const response = await auth.handler( + request( + path, + mode === 'single' ? { sessionToken: previous.token } : { userId }, + administrator.cookie + ) + ) + expect(response.status).toBe(200) + expect(await sessionCount()).toBe(remaining) + const entry = await expectEvent('auth.sessions_revoked', administrator.id) + expect(entry?.metadata).toMatchObject({ + revocationMode: mode, + sessionsMatchedBefore: matched, + }) + const entries = JSON.stringify(await history()) + expect(entries).not.toContain(previous.token) + expect(entries).not.toContain(current.token) + expect(entries).not.toContain(administrator.token) + } + ) +}) diff --git a/apps/sim/lib/auth/lifecycle-audit.ts b/apps/sim/lib/auth/lifecycle-audit.ts new file mode 100644 index 00000000000..ef93e747370 --- /dev/null +++ b/apps/sim/lib/auth/lifecycle-audit.ts @@ -0,0 +1,368 @@ +import { AuditAction, type AuditLogParams, AuditResourceType, recordAudit } from '@sim/audit' +import { db } from '@sim/db' +import { foldedEmail, member, session, ssoProvider, user } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { isAPIError } from 'better-auth/api' +import { and, count, eq, gt, ne } from 'drizzle-orm' +import { isSignInProviderAllowed } from '@/lib/auth/constants' +import { isSsoCallbackPath } from '@/lib/auth/sso/callback-provider' + +const logger = createLogger('AuthenticationAudit') + +type AuthenticationMethod = 'email_password' | 'email_otp' | 'social' | 'sso' +type AuthenticationFailureReason = + | 'credential_rejected' + | 'access_denied' + | 'invalid_request' + | 'authentication_error' + | 'provider_rejected' + | 'sso_admission_denied' + +interface AuthenticationAuditInput { + path: string + userId?: string + email?: unknown + logoutUserId?: string + providerId?: string + returned?: unknown + redirectLocation?: string | null + failureReason?: AuthenticationFailureReason + ssoAdmissionCompleted?: boolean + request?: AuditLogParams['request'] +} + +function authenticationMethod(path: string): AuthenticationMethod | null { + if (path === '/sign-in/email' || path === '/sign-up/email') return 'email_password' + if (path === '/sign-in/email-otp') return 'email_otp' + if (isSsoCallbackPath(path) || path === '/sign-in/sso') return 'sso' + if ( + path === '/sign-in/social' || + path === '/sign-in/oauth2' || + path.startsWith('/callback/') || + path.startsWith('/oauth2/callback/') + ) { + return 'social' + } + return null +} + +function failureReason(statusCode: number): AuthenticationFailureReason { + if (statusCode === 401) return 'credential_rejected' + if (statusCode === 403) return 'access_denied' + if (statusCode >= 400 && statusCode < 500) return 'invalid_request' + return 'authentication_error' +} + +function isFailureRedirect(location: string | null | undefined): boolean { + if (!location) return false + try { + return new URL(location, 'https://auth.invalid').searchParams.has('error') + } catch { + return false + } +} + +async function loadSubject(input: { userId?: string; email?: unknown }) { + const email = typeof input.email === 'string' ? input.email.trim().toLowerCase() : undefined + if (!input.userId && !email) return undefined + const [subject] = await db + .select({ + id: user.id, + name: user.name, + email: user.email, + organizationId: member.organizationId, + }) + .from(user) + .leftJoin(member, eq(member.userId, user.id)) + .where(input.userId ? eq(user.id, input.userId) : eq(foldedEmail(user.email), email ?? '')) + .limit(1) + return subject +} + +/** Records final authentication outcomes without persisting request bodies or bearer credentials. */ +export async function recordAuthenticationAudit(input: AuthenticationAuditInput): Promise { + const statusCode = isAPIError(input.returned) ? input.returned.statusCode : undefined + if (input.path === '/request-password-reset') { + if ((statusCode && statusCode >= 400) || input.returned instanceof Error) return + await recordPasswordResetAudit({ + kind: 'requested', + email: typeof input.email === 'string' ? input.email : undefined, + request: input.request, + }) + return + } + const method = authenticationMethod(input.path) + const reason = + input.failureReason ?? + (statusCode && statusCode >= 400 + ? failureReason(statusCode) + : input.returned instanceof Error && !isAPIError(input.returned) + ? 'authentication_error' + : method && !input.userId && isFailureRedirect(input.redirectLocation) + ? 'provider_rejected' + : undefined) + const outcome = reason + ? 'failure' + : input.path === '/sign-out' && input.logoutUserId + ? 'logout' + : method && input.userId + ? 'success' + : undefined + if (!outcome || (!method && outcome !== 'logout')) return + if (outcome === 'failure' && input.path === '/sign-up/email') return + if (outcome === 'success' && isSsoCallbackPath(input.path) && !input.ssoAdmissionCompleted) { + return + } + + try { + const subject = await loadSubject({ + userId: outcome === 'logout' ? input.logoutUserId : input.userId, + email: input.email, + }) + if (!subject && outcome !== 'failure') return + let organizationId = subject?.organizationId + let providerId = isSignInProviderAllowed(input.providerId) ? input.providerId : undefined + if (method === 'sso' && input.providerId && /^[a-zA-Z0-9_.:-]{1,128}$/.test(input.providerId)) { + const [provider] = await db + .select({ organizationId: ssoProvider.organizationId, providerId: ssoProvider.providerId }) + .from(ssoProvider) + .where(eq(ssoProvider.providerId, input.providerId)) + .limit(1) + providerId = provider?.providerId + if (outcome === 'failure' || !organizationId) { + organizationId = provider?.organizationId ?? organizationId + } + } + const authenticatedActor = outcome !== 'failure' ? subject : undefined + recordAudit({ + actorId: authenticatedActor?.id ?? null, + actorName: authenticatedActor?.name ?? 'Unauthenticated requester', + actorEmail: authenticatedActor?.email ?? null, + action: + outcome === 'success' + ? AuditAction.AUTH_LOGIN_SUCCEEDED + : outcome === 'logout' + ? AuditAction.AUTH_LOGOUT + : AuditAction.AUTH_LOGIN_FAILED, + resourceType: AuditResourceType.USER, + resourceId: subject?.id, + resourceName: subject?.email, + description: + outcome === 'success' + ? 'User signed in' + : outcome === 'logout' + ? 'User signed out' + : 'Authentication attempt rejected', + metadata: { + outcome, + ...(method ? { authenticationMethod: method } : {}), + ...(organizationId ? { organizationId } : {}), + ...(providerId ? { providerId } : {}), + ...(reason ? { failureReason: reason } : {}), + ...(statusCode && statusCode >= 400 ? { statusCode } : {}), + ...(outcome === 'failure' && subject ? { targetUserId: subject.id } : {}), + }, + request: input.request, + }) + } catch { + logger.warn('Failed to prepare authentication audit event', { outcome }) + } +} + +/** Attributes a reset request to its anonymous requester and a completed reset to its user. */ +export async function recordPasswordResetAudit(input: { + kind: 'requested' | 'completed' + userId?: string + email?: string + request?: AuditLogParams['request'] +}): Promise { + try { + const subject = await loadSubject(input) + if (!subject) return + const completed = input.kind === 'completed' + recordAudit({ + actorId: completed ? subject.id : null, + actorName: completed ? subject.name : 'Unauthenticated requester', + actorEmail: completed ? subject.email : null, + action: completed ? AuditAction.PASSWORD_RESET : AuditAction.PASSWORD_RESET_REQUESTED, + resourceType: AuditResourceType.PASSWORD, + resourceId: subject.id, + resourceName: subject.email, + description: completed ? 'Password reset completed' : 'Password reset requested', + metadata: { + outcome: 'success', + ...(subject.organizationId ? { organizationId: subject.organizationId } : {}), + ...(!completed ? { targetUserId: subject.id } : {}), + }, + request: input.request, + }) + } catch { + logger.warn('Failed to prepare password reset audit event', { kind: input.kind }) + } +} + +interface SecurityAuditActor { + userId: string + sessionId: string + role?: unknown + impersonatedBy?: unknown +} + +export interface AuthenticationSecurityAuditSnapshot { + kind: 'impersonation_started' | 'impersonation_ended' | 'sessions_revoked' + actorUserId: string + targetUserId: string + administrative: boolean + revocationMode?: 'single' | 'all' | 'other' + sessionsMatchedBefore?: number +} + +const SECURITY_MUTATION_PATHS = new Set([ + '/admin/impersonate-user', + '/admin/stop-impersonating', + '/admin/revoke-user-session', + '/admin/revoke-user-sessions', + '/revoke-session', + '/revoke-sessions', + '/revoke-other-sessions', +]) + +/** Captures canonical session references and aggregate counts before revocation removes them. */ +export async function captureAuthenticationSecurityAudit(input: { + path: string + resolveActor: () => Promise + requestedUserId?: unknown + requestedToken?: unknown +}): Promise { + if (!SECURITY_MUTATION_PATHS.has(input.path)) return undefined + try { + const actor = await input.resolveActor() + if (!actor) return undefined + if (input.path === '/admin/stop-impersonating') { + if (typeof actor.impersonatedBy !== 'string' || !actor.impersonatedBy) return undefined + return { + kind: 'impersonation_ended', + actorUserId: actor.impersonatedBy, + targetUserId: actor.userId, + administrative: true, + } + } + + const administrative = input.path.startsWith('/admin/') + if ( + administrative && + !(typeof actor.role === 'string' && actor.role.split(',').includes('admin')) + ) { + return undefined + } + if (input.path === '/admin/impersonate-user') { + if (typeof input.requestedUserId !== 'string') return undefined + return { + kind: 'impersonation_started', + actorUserId: actor.userId, + targetUserId: input.requestedUserId, + administrative, + } + } + + const single = input.path === '/revoke-session' || input.path === '/admin/revoke-user-session' + if (single) { + if (typeof input.requestedToken !== 'string') return undefined + const [target] = await db + .select({ userId: session.userId }) + .from(session) + .where( + and( + eq(session.token, input.requestedToken), + administrative ? undefined : eq(session.userId, actor.userId) + ) + ) + .limit(1) + if (!target) return undefined + return { + kind: 'sessions_revoked', + actorUserId: actor.userId, + targetUserId: target.userId, + administrative, + revocationMode: 'single', + sessionsMatchedBefore: 1, + } + } + + const targetUserId = administrative ? input.requestedUserId : actor.userId + if (typeof targetUserId !== 'string') return undefined + const other = input.path === '/revoke-other-sessions' + const [matched] = await db + .select({ total: count() }) + .from(session) + .where( + and( + eq(session.userId, targetUserId), + other ? ne(session.id, actor.sessionId) : undefined, + other ? gt(session.expiresAt, new Date()) : undefined + ) + ) + return { + kind: 'sessions_revoked', + actorUserId: actor.userId, + targetUserId, + administrative, + revocationMode: other ? 'other' : 'all', + sessionsMatchedBefore: matched?.total ?? 0, + } + } catch { + logger.warn('Failed to capture authentication security audit context') + return undefined + } +} + +/** Records accepted security mutations in the target organization with the actual operator. */ +export async function recordAuthenticationSecurityAudit(input: { + snapshot?: AuthenticationSecurityAuditSnapshot + returned?: unknown + request?: AuditLogParams['request'] +}): Promise { + if (!input.snapshot || input.returned instanceof Error) return + if (isAPIError(input.returned) && input.returned.statusCode >= 400) return + const snapshot = input.snapshot + try { + const [actor, target] = await Promise.all([ + loadSubject({ userId: snapshot.actorUserId }), + loadSubject({ userId: snapshot.targetUserId }), + ]) + if (!actor || !target) return + recordAudit({ + actorId: actor.id, + actorName: actor.name, + actorEmail: actor.email, + action: + snapshot.kind === 'impersonation_started' + ? AuditAction.AUTH_IMPERSONATION_STARTED + : snapshot.kind === 'impersonation_ended' + ? AuditAction.AUTH_IMPERSONATION_ENDED + : AuditAction.AUTH_SESSIONS_REVOKED, + resourceType: AuditResourceType.USER, + resourceId: target.id, + resourceName: target.email, + description: + snapshot.kind === 'impersonation_started' + ? 'Administrator started impersonating a user' + : snapshot.kind === 'impersonation_ended' + ? 'Administrator stopped impersonating a user' + : 'Processed session revocation request', + metadata: { + outcome: 'success', + targetUserId: target.id, + administrative: snapshot.administrative, + ...(target.organizationId ? { organizationId: target.organizationId } : {}), + ...(snapshot.revocationMode ? { revocationMode: snapshot.revocationMode } : {}), + ...(snapshot.sessionsMatchedBefore !== undefined + ? { sessionsMatchedBefore: snapshot.sessionsMatchedBefore } + : {}), + }, + request: input.request, + }) + } catch { + logger.warn('Failed to prepare authentication security audit event', { kind: snapshot.kind }) + } +} diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 77160997596..95fbfcc1259 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -340,6 +340,12 @@ export interface SecureFetchOptions { method?: string headers?: Record body?: string | Buffer | Uint8Array + /** + * Raw origin-form path and query for authenticated protocols that sign the request target. + * Preserves dot segments, keeps the URL's host, disables redirects, and fails closed when + * the selected gateway cannot preserve this target. + */ + requestTarget?: string timeout?: number maxRedirects?: number /** @@ -1127,7 +1133,17 @@ export async function secureFetchWithPinnedIP( options: SecureFetchOptions, redirectCount = 0 ): Promise { - const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS + const requestTarget = options.requestTarget + if ( + requestTarget !== undefined && + (!requestTarget.startsWith('/') || + requestTarget.includes('#') || + /[^\x21-\x7e]/.test(requestTarget)) + ) { + throw new Error('Invalid signed HTTP request target') + } + const maxRedirects = + requestTarget === undefined ? (options.maxRedirects ?? DEFAULT_MAX_REDIRECTS) : 0 const requestedMaxResponseBytes = options.maxResponseBytes const maxResponseBytes = typeof requestedMaxResponseBytes === 'number' && requestedMaxResponseBytes > 0 @@ -1140,6 +1156,10 @@ export async function secureFetchWithPinnedIP( proxyUrl: options.proxyUrl, }) const outboundDispatcher = await transport.selectDispatcher() + if (requestTarget !== undefined && outboundDispatcher) { + await transport.destroy() + throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') + } return new Promise((resolve, reject) => { const parsed = new URL(url) @@ -1189,7 +1209,7 @@ export async function secureFetchWithPinnedIP( const requestOptions: http.RequestOptions = { hostname: parsed.hostname, port, - path: parsed.pathname + parsed.search, + path: requestTarget ?? parsed.pathname + parsed.search, method: options.method || 'GET', headers: sanitizedHeaders, agent, diff --git a/apps/sim/lib/data-drains/application/use-cases.ts b/apps/sim/lib/data-drains/application/use-cases.ts index f69ca519601..0dca726213b 100644 --- a/apps/sim/lib/data-drains/application/use-cases.ts +++ b/apps/sim/lib/data-drains/application/use-cases.ts @@ -3,7 +3,7 @@ import { type Principal, requirePrincipalSubjectUserId } from '@sim/auth/princip import { db } from '@sim/db' import { dataDrainRuns, dataDrains } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getPostgresErrorCode, toError } from '@sim/utils/errors' +import { getPostgresErrorCode } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { omit } from '@sim/utils/object' import { and, asc, desc, eq, ne } from 'drizzle-orm' @@ -15,7 +15,6 @@ import { } from '@/lib/core/application/authorized-workspace-use-case' import type { OperationUseCase } from '@/lib/core/application/operation' import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' -import { getJobQueue } from '@/lib/core/async-jobs' import { isBillingEnabled, isDataDrainsEnabled } from '@/lib/core/config/env-flags' import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { @@ -28,6 +27,8 @@ import { } from '@/lib/data-drains/application/operations' import { getDestination } from '@/lib/data-drains/destinations/registry' import { decryptCredentials, encryptCredentials } from '@/lib/data-drains/encryption' +import { enqueueDrain } from '@/lib/data-drains/enqueue' +import { createCredentialErrorRedactor } from '@/lib/data-drains/errors' import { type CreateDataDrainInput, createDataDrainBodySchema, @@ -350,12 +351,7 @@ export const runDataDrain = defineDrainUseCase({ .limit(1) if (inFlight) throw new OrchestrationError('conflict', 'A run is already in progress for this drain') - const queue = await getJobQueue() - const jobId = await queue.enqueue( - 'run-data-drain', - { drainId: drain.id, trigger: 'manual' }, - { concurrencyKey: `data-drain:${drain.id}` } - ) + const jobId = await enqueueDrain(drain.id, 'manual') return { jobId, drain: publicDrain(drain) } }, projectAudit: ({ result }) => ({ @@ -378,19 +374,20 @@ export const testDataDrain = defineDrainUseCase({ 'validation', `Destination '${drain.destinationType}' does not support connection testing` ) - const config = destination.configSchema.parse(drain.destinationConfig) - const credentials = destination.credentialsSchema.parse( - await decryptCredentials(drain.destinationCredentials) - ) const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), 10_000) + let redactError = createCredentialErrorRedactor() try { + const config = destination.configSchema.parse(drain.destinationConfig) + const savedCredentials = await decryptCredentials(drain.destinationCredentials) + redactError = createCredentialErrorRedactor(savedCredentials) + const credentials = destination.credentialsSchema.parse(savedCredentials) await runWithOutboundOrganization(drain.organizationId, () => test({ config, credentials, signal: controller.signal }) ) return { drain: publicDrain(drain), ok: true as const } } catch (error) { - const message = toError(error).message + const message = redactError(error) logger.warn('Data drain test connection failed', { drainId: drain.id, destinationType: drain.destinationType, diff --git a/apps/sim/lib/data-drains/destinations/azure_blob.ts b/apps/sim/lib/data-drains/destinations/azure_blob.ts index d038a84aca4..1ce9a548a72 100644 --- a/apps/sim/lib/data-drains/destinations/azure_blob.ts +++ b/apps/sim/lib/data-drains/destinations/azure_blob.ts @@ -125,7 +125,10 @@ async function withAzureErrorContext(action: string, fn: () => Promise): P if (isAzureRestError(error)) { const status = error.statusCode const code = error.code - logger.warn('Azure Blob operation failed', { action, code, status }) + logger.warn('Azure Blob operation failed', { + action, + ...(typeof status === 'number' ? { status } : {}), + }) throw new Error( `Azure Blob ${action} failed (${code ?? 'Error'}${status ? ` ${status}` : ''}): ${error.message ?? ''}`, { cause: error } @@ -156,12 +159,11 @@ export const azureBlobDestination: DrainDestination< ) try { await blockBlobClient.deleteIfExists({ abortSignal: signal }) - } catch (cleanupError) { + } catch { logger.debug('Azure Blob test write probe cleanup failed (non-fatal)', { accountName: config.accountName, containerName: config.containerName, blobName: probeName, - error: cleanupError, }) } }, diff --git a/apps/sim/lib/data-drains/destinations/bigquery.ts b/apps/sim/lib/data-drains/destinations/bigquery.ts index da342c66f44..b4dc41b841a 100644 --- a/apps/sim/lib/data-drains/destinations/bigquery.ts +++ b/apps/sim/lib/data-drains/destinations/bigquery.ts @@ -1,6 +1,5 @@ import { createHash } from 'node:crypto' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { JWT } from 'google-auth-library' @@ -124,7 +123,6 @@ async function postInsertAll( } catch (error) { logger.warn('BigQuery request failed', { table: `${input.config.projectId}.${input.config.datasetId}.${input.config.tableId}`, - error: toError(error).message, }) throw error } @@ -236,7 +234,6 @@ async function insertAll(input: InsertAllInput): Promise { logger.warn('BigQuery insertAll network error; retrying', { attempt, retryAfterMs, - error: toError(error).message, }) await interruptibleSleep(retryAfterMs, input.signal) if (input.signal.aborted) throw input.signal.reason ?? new Error('Aborted') @@ -260,7 +257,7 @@ async function insertAll(input: InsertAllInput): Promise { table: `${input.config.projectId}.${input.config.datasetId}.${input.config.tableId}`, succeededRows: succeeded, failedRows: failed, - failedIndices: failedIndices.slice(0, 20), + failedIndices: failedIndices.filter((index) => typeof index === 'number').slice(0, 20), }) const summary = result.insertErrors .slice(0, 3) diff --git a/apps/sim/lib/data-drains/destinations/datadog.test.ts b/apps/sim/lib/data-drains/destinations/datadog.test.ts index 0715bd6f5b8..eaf57aa8b89 100644 --- a/apps/sim/lib/data-drains/destinations/datadog.test.ts +++ b/apps/sim/lib/data-drains/destinations/datadog.test.ts @@ -101,8 +101,7 @@ describe('datadogDestination', () => { it('throws with the entry index when a single entry exceeds 1 MB', async () => { const session = datadogDestination.openSession({ config, credentials }) - // Two entries; the second exceeds the 1 MB per-entry limit. - const huge = 'x'.repeat(1024 * 1024 + 10) + const huge = 'x'.repeat(500_000) const body = Buffer.from( `${JSON.stringify({ id: 'small' })}\n${JSON.stringify({ blob: huge })}\n`, 'utf8' @@ -114,11 +113,72 @@ describe('datadogDestination', () => { metadata: meta(0), signal: new AbortController().signal, }) - ).rejects.toThrow(/entry at index 1 is .* exceeds the 1048576-byte per-entry limit/) + ).rejects.toThrow(/entry at index 1 is .* exceeds the 1000000-byte per-entry limit/) expect(fetchMock).not.toHaveBeenCalled() await session.close() }) + it.each([ + { count: 6, paddingBytes: 425_000, limit: 'uncompressed bytes' }, + { count: 1001, paddingBytes: 0, limit: 'entry count' }, + ])( + 'splits $limit overflow without losing or reordering entries', + async ({ count, paddingBytes }) => { + const rows = Array.from({ length: count }, (_, index) => ({ + id: `event-${index}`, + detail: 'x'.repeat(paddingBytes), + })) + const session = datadogDestination.openSession({ config, credentials }) + try { + await session.deliver({ + body: Buffer.from(`${rows.map((row) => JSON.stringify(row)).join('\n')}\n`), + contentType: 'application/x-ndjson', + metadata: { ...meta(0), rowCount: rows.length }, + signal: new AbortController().signal, + }) + const deliveredIds: string[] = [] + for (const call of fetchMock.mock.calls) { + const init = call[1] as RequestInit + const headers = init.headers as Record + const payload = + headers['Content-Encoding'] === 'gzip' + ? gunzipSync(init.body as Uint8Array).toString('utf8') + : (init.body as string) + expect(Buffer.byteLength(payload, 'utf8')).toBeLessThanOrEqual(5_000_000) + const entries = JSON.parse(payload) as { id: string }[] + expect(entries.length).toBeLessThanOrEqual(1000) + deliveredIds.push(...entries.map((entry) => entry.id)) + } + expect(deliveredIds).toEqual(rows.map((row) => row.id)) + } finally { + await session.close() + } + } + ) + + it('redacts the configured API key when the provider echoes it in an error', async () => { + fetchMock.mockResolvedValueOnce( + new Response(`Invalid API key: ${credentials.apiKey}`, { status: 403 }) + ) + const session = datadogDestination.openSession({ config, credentials }) + try { + const result = await session + .deliver({ + body: Buffer.from('{"id":"event"}\n'), + contentType: 'application/x-ndjson', + metadata: meta(0), + signal: new AbortController().signal, + }) + .catch((error: unknown) => error) + expect(result).toBeInstanceOf(Error) + if (!(result instanceof Error)) throw new Error('Expected delivery to fail') + expect(result.message).toContain('HTTP 403') + expect(result.message).not.toContain(credentials.apiKey) + } finally { + await session.close() + } + }) + it('gzips payloads larger than 1KB and sets Content-Encoding: gzip', async () => { const session = datadogDestination.openSession({ config, credentials }) // Build > 1KB raw payload; padding string is JSON-safe. diff --git a/apps/sim/lib/data-drains/destinations/datadog.ts b/apps/sim/lib/data-drains/destinations/datadog.ts index 801aec3574b..d57568e3e7f 100644 --- a/apps/sim/lib/data-drains/destinations/datadog.ts +++ b/apps/sim/lib/data-drains/destinations/datadog.ts @@ -4,6 +4,7 @@ import { toError } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { z } from 'zod' +import { redactExactSensitiveValues } from '@/lib/core/security/redaction' import { parseNdjsonObjects } from '@/lib/data-drains/destinations/utils' import type { DeliveryMetadata, DrainDestination } from '@/lib/data-drains/types' @@ -25,9 +26,9 @@ const SITE_HOSTS: Record = { const MAX_ATTEMPTS = 4 const PER_ATTEMPT_TIMEOUT_MS = 30_000 -const MAX_UNCOMPRESSED_BYTES = 5 * 1024 * 1024 +const MAX_UNCOMPRESSED_BYTES = 5_000_000 const MAX_WIRE_BYTES = 6 * 1024 * 1024 -const MAX_ENTRY_BYTES = 1024 * 1024 +const MAX_ENTRY_BYTES = 1_000_000 const MAX_ENTRIES_PER_REQUEST = 1000 const GZIP_THRESHOLD_BYTES = 1024 @@ -151,6 +152,7 @@ function buildRequestBody(payload: string, apiKey: string): PreparedBody { async function postWithRetries(input: PostInput): Promise { const { body, headers } = input.prepared + const apiKey = headers['DD-API-KEY'] let lastError: unknown for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { if (input.signal.aborted) throw input.signal.reason ?? new Error('Aborted') @@ -166,8 +168,9 @@ async function postWithRetries(input: PostInput): Promise { signal: perAttempt, }) } catch (error) { - lastError = error - logger.debug('Datadog request failed', { attempt, error: toError(error).message }) + const message = redactExactSensitiveValues(toError(error).message, [apiKey]) + lastError = new Error(message) + logger.debug('Datadog request failed', { attempt }) } if (response) { if (response.ok) { @@ -177,7 +180,9 @@ async function postWithRetries(input: PostInput): Promise { } if (!isRetryableStatus(response.status)) { const text = await response.text().catch(() => '') - throw new Error(`Datadog responded with HTTP ${response.status}: ${text}`) + throw new Error( + `Datadog responded with HTTP ${response.status}: ${redactExactSensitiveValues(text, [apiKey])}` + ) } lastError = new Error(`Datadog responded with HTTP ${response.status}`) retryAfterMs = parseRetryAfter(response.headers.get('retry-after')) @@ -222,42 +227,55 @@ export const datadogDestination: DrainDestination< async deliver({ body, metadata, signal }) { const rows = parseNdjsonObjects(body) const entries = buildEntries(rows, config, metadata) - if (entries.length > MAX_ENTRIES_PER_REQUEST) { - throw new Error( - `Datadog chunk has ${entries.length} entries, exceeds the ${MAX_ENTRIES_PER_REQUEST} per-request limit` - ) - } - for (let i = 0; i < entries.length; i++) { - const entryBytes = Buffer.byteLength(JSON.stringify(entries[i]), 'utf8') + const serializedEntries = entries.map((entry, index) => { + const json = JSON.stringify(entry) + const entryBytes = Buffer.byteLength(json, 'utf8') if (entryBytes > MAX_ENTRY_BYTES) { throw new Error( - `Datadog entry at index ${i} is ${entryBytes} bytes, exceeds the ${MAX_ENTRY_BYTES}-byte per-entry limit` + `Datadog entry at index ${index} is ${entryBytes} bytes, exceeds the ${MAX_ENTRY_BYTES}-byte per-entry limit` ) } + return { json, bytes: entryBytes } + }) + let batch: string[] = [] + let batchBytes = 2 + let rawBytes = 0 + let wireBytes = 0 + let requestId: string | null = null + let requests = 0 + const sendBatch = async () => { + const prepared = buildRequestBody(`[${batch.join(',')}]`, credentials.apiKey) + if (prepared.wireBytes > MAX_WIRE_BYTES) { + throw new Error( + `Datadog payload is ${prepared.wireBytes} bytes on the wire, exceeds the ${MAX_WIRE_BYTES}-byte defensive wire-size cap` + ) + } + const response = await postWithRetries({ url, prepared, signal }) + requestId = response.headers.get('dd-request-id') + rawBytes += prepared.rawBytes + wireBytes += prepared.wireBytes + requests++ + batch = [] + batchBytes = 2 } - const payload = JSON.stringify(entries) - const prepared = buildRequestBody(payload, credentials.apiKey) - if (prepared.rawBytes > MAX_UNCOMPRESSED_BYTES) { - throw new Error( - `Datadog payload is ${prepared.rawBytes} bytes uncompressed, exceeds the ${MAX_UNCOMPRESSED_BYTES}-byte per-request limit` - ) - } - if (prepared.wireBytes > MAX_WIRE_BYTES) { - throw new Error( - `Datadog payload is ${prepared.wireBytes} bytes on the wire, exceeds the ${MAX_WIRE_BYTES}-byte defensive wire-size cap` - ) + for (const entry of serializedEntries) { + const addedBytes = entry.bytes + (batch.length > 0 ? 1 : 0) + if ( + batch.length >= MAX_ENTRIES_PER_REQUEST || + batchBytes + addedBytes > MAX_UNCOMPRESSED_BYTES + ) { + await sendBatch() + } + batchBytes += entry.bytes + (batch.length > 0 ? 1 : 0) + batch.push(entry.json) } - const response = await postWithRetries({ - url, - prepared, - signal, - }) - const requestId = response.headers.get('dd-request-id') ?? null + if (batch.length > 0 || serializedEntries.length === 0) await sendBatch() logger.debug('Datadog chunk delivered', { site: config.site, rows: entries.length, - rawBytes: prepared.rawBytes, - wireBytes: prepared.wireBytes, + requests, + rawBytes, + wireBytes, }) return { locator: requestId diff --git a/apps/sim/lib/data-drains/destinations/gcs.ts b/apps/sim/lib/data-drains/destinations/gcs.ts index 70a6d576915..e7cfe734536 100644 --- a/apps/sim/lib/data-drains/destinations/gcs.ts +++ b/apps/sim/lib/data-drains/destinations/gcs.ts @@ -1,5 +1,4 @@ import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { generateShortId } from '@sim/utils/id' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' @@ -152,7 +151,6 @@ async function fetchWithRetry(input: RetryRequestInput): Promise { action: input.action, attempt, bucket: input.bucket, - error: toError(error).message, }) if (attempt < MAX_ATTEMPTS) { await interruptibleSleep(backoffWithJitter(attempt, null), input.signal) @@ -277,11 +275,10 @@ export const gcsDestination: DrainDestination { + vi.useFakeTimers() + setEnvFlags({ isHosted: true }) + requests.length = 0 + vi.spyOn(https, 'request').mockImplementation((( + options: RequestOptions, + onResponse: (response: IncomingMessage) => void + ) => { + requests.push(options) + const request = new Writable({ + write(_chunk, _encoding, callback) { + callback() + }, + }) + request.on('finish', () => { + const response = Object.assign(Readable.from([]), { statusCode: 200, headers: {} }) + onResponse(response as IncomingMessage) + }) + return request as ClientRequest + }) as typeof https.request) +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('S3 drain transport security', () => { + it('preserves the signed object request target through the shared egress guard', async () => { + const target = '/exports/../raw/audit_logs/event.ndjson?x-id=PutObject' + const response = await secureFetchWithPinnedIP( + `https://audit-bucket.storage.example.com${target}`, + '1.1.1.1', + { + profile: 'configuredEndpoint', + requestTarget: target, + method: 'PUT', + headers: { authorization: 'AWS4-HMAC-SHA256 signed-request' }, + body: '{"id":"event"}\n', + } + ) + await response.text() + expect(requests[0].path).toBe(target) + expect(requests[0].hostname).toBe('audit-bucket.storage.example.com') + expect(requests[0].headers).toMatchObject({ + authorization: 'AWS4-HMAC-SHA256 signed-request', + }) + }) + + it.each(['https://other.example.com/object', '/object#fragment', '/object\r\nX-Injected: yes'])( + 'rejects an unsafe signed request target %j before making a connection', + async (requestTarget) => { + await expect( + secureFetchWithPinnedIP('https://storage.example.com', '1.1.1.1', { + profile: 'configuredEndpoint', + requestTarget, + }) + ).rejects.toThrow(/request target/i) + expect(requests).toHaveLength(0) + } + ) + + it('rejects a private bucket host even when the configured endpoint resolves publicly', async () => { + vi.spyOn(dns, 'lookup').mockImplementation(async (host) => [ + { address: host.startsWith('audit-bucket.') ? '10.0.0.5' : '1.1.1.1', family: 4 }, + ]) + const session = s3Destination.openSession({ config, credentials }) + try { + await expect( + session.deliver({ + body: Buffer.from('{"id":"event"}\n'), + contentType: 'application/x-ndjson', + metadata, + signal: new AbortController().signal, + }) + ).rejects.toThrow(/private|reserved/) + expect(requests).toHaveLength(0) + } finally { + await session.close() + } + }) + + it('pins the checked address on the signed SDK request', async () => { + const lookup = vi.spyOn(dns, 'lookup').mockResolvedValue([{ address: '1.1.1.1', family: 4 }]) + const session = s3Destination.openSession({ config, credentials }) + try { + await session.deliver({ + body: Buffer.from('{"id":"event"}\n'), + contentType: 'application/x-ndjson', + metadata, + signal: new AbortController().signal, + }) + const [request] = requests + expect(request.hostname ?? request.host).toBe('audit-bucket.storage.example.com') + expect(request.headers).toMatchObject({ + host: 'audit-bucket.storage.example.com', + authorization: expect.stringContaining('AWS4-HMAC-SHA256'), + }) + lookup.mockResolvedValue([{ address: '10.0.0.5', family: 4 }]) + const connectionLookup = request.lookup + expect(connectionLookup).toBeTypeOf('function') + if (!connectionLookup) throw new Error('SDK transport did not pin its DNS lookup') + const pinnedAddress = await new Promise((resolve, reject) => { + connectionLookup('audit-bucket.storage.example.com', { all: false }, (error, address) => { + if (error) reject(error) + else if (typeof address === 'string') resolve(address) + else reject(new Error('Expected one pinned address')) + }) + }) + expect(pinnedAddress).toBe('1.1.1.1') + } finally { + await session.close() + } + }) +}) diff --git a/apps/sim/lib/data-drains/destinations/s3.ts b/apps/sim/lib/data-drains/destinations/s3.ts index 7d464626547..11a4daf5c2c 100644 --- a/apps/sim/lib/data-drains/destinations/s3.ts +++ b/apps/sim/lib/data-drains/destinations/s3.ts @@ -2,13 +2,18 @@ import { DeleteObjectCommand, PutObjectCommand, S3Client, + type S3ClientConfig, type S3ServiceException, } from '@aws-sdk/client-s3' import { createLogger } from '@sim/logger' import { generateShortId } from '@sim/utils/id' import { z } from 'zod' import { validateExternalUrl } from '@/lib/core/security/input-validation' -import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' +import { + createPinnedConnectionPool, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import { buildObjectKey, normalizePrefix } from '@/lib/data-drains/destinations/utils' import type { DrainDestination } from '@/lib/data-drains/types' @@ -22,6 +27,13 @@ const AWS_REGION_RE = /^[a-z]{2,}(-[a-z]+)+-\d+$/ /** Cap is over key + value bytes only (no `x-amz-meta-` prefix). */ const MAX_S3_METADATA_BYTES = 2 * 1024 const MAX_S3_KEY_BYTES = 1024 +const REQUEST_TIMEOUT_MS = 60_000 +const MAX_RESPONSE_BYTES = 256 * 1024 + +type S3RequestHandler = Extract< + NonNullable, + { updateHttpClientConfig: unknown; httpHandlerConfigs: unknown } +> const s3BucketSchema = z .string() @@ -104,6 +116,84 @@ const s3CredentialsSchema = z.object({ export type S3DestinationConfig = z.infer export type S3DestinationCredentials = z.infer +function createRequestHandler(): S3RequestHandler { + const connections = createPinnedConnectionPool() + return { + async handle( + request: Parameters[0], + options: Parameters[1] + ) { + if (request.protocol !== 'https:') throw new Error('S3 requests must use HTTPS') + const callerSignal = options?.abortSignal + if (callerSignal && !(callerSignal instanceof AbortSignal)) { + throw new Error('S3 transport requires a native AbortSignal') + } + const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS) + const signal = callerSignal ? AbortSignal.any([callerSignal, timeout]) : timeout + signal.throwIfAborted() + const hostname = request.hostname + const authority = hostname.includes(':') ? `[${hostname}]` : hostname + const origin = `https://${authority}${request.port ? `:${request.port}` : ''}` + const validation = await validateUrlWithDNS(origin, 'S3 endpoint', 'configuredEndpoint', { + signal, + }) + if (!validation.isValid) throw new Error(validation.error) + const body: unknown = request.body + if (body !== undefined && typeof body !== 'string' && !(body instanceof Uint8Array)) { + throw new Error('Unsupported S3 request body') + } + const encodeQueryComponent = (value: string) => + encodeURIComponent(value).replace( + /[!'()*]/g, + (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}` + ) + const query = request.query ?? {} + const queryParts: string[] = [] + for (const key of Object.keys(query).sort()) { + const value = query[key] + const encodedKey = encodeQueryComponent(key) + if (value === null) queryParts.push(encodedKey) + else { + for (const item of Array.isArray(value) ? value : [value]) { + queryParts.push(`${encodedKey}=${encodeQueryComponent(item)}`) + } + } + } + const search = queryParts.join('&') + const requestTarget = `${request.path}${search ? `?${search}` : ''}` + const response = await secureFetchWithPinnedIP( + `${origin}${requestTarget}`, + validation.resolvedIP, + { + profile: 'configuredEndpoint', + requestTarget, + method: request.method, + headers: request.headers, + body, + signal, + timeout: REQUEST_TIMEOUT_MS, + maxResponseBytes: MAX_RESPONSE_BYTES, + connectionPool: connections, + } + ) + return { + response: { + statusCode: response.status, + headers: response.headers.toRecord(), + body: Buffer.from(await response.arrayBuffer()), + }, + } + }, + destroy() { + connections.destroy() + }, + updateHttpClientConfig() {}, + httpHandlerConfigs() { + return {} + }, + } +} + function buildClient(config: S3DestinationConfig, credentials: S3DestinationCredentials): S3Client { return new S3Client({ region: config.region, @@ -113,6 +203,7 @@ function buildClient(config: S3DestinationConfig, credentials: S3DestinationCred }, endpoint: config.endpoint, forcePathStyle: config.forcePathStyle ?? false, + requestHandler: createRequestHandler(), }) } @@ -125,15 +216,6 @@ function isS3ServiceException(error: unknown): error is S3ServiceException { ) } -/** DNS-aware SSRF check: catches hostnames that resolve to internal IPs (the schema check only catches IP literals). */ -async function assertEndpointIsPublic(endpoint: string | undefined): Promise { - if (!endpoint) return - const result = await validateUrlWithDNS(endpoint, 'endpoint', 'configuredEndpoint') - if (!result.isValid) { - throw new Error(result.error) - } -} - /** * Surfaces actionable S3 SDK error codes (`AccessDenied`, `NoSuchBucket`, * `InvalidAccessKeyId`, `SignatureDoesNotMatch`, ...) and preserves the @@ -146,8 +228,7 @@ async function withS3ErrorContext(action: string, fn: () => Promise): Prom if (isS3ServiceException(error)) { const code = error.name const status = error.$metadata?.httpStatusCode - const requestId = error.$metadata?.requestId - logger.warn('S3 operation failed', { action, code, status, requestId }) + logger.warn('S3 operation failed', { action, status }) /** Preserve SDK error as `cause` so callers can still branch on `code` / `$metadata`. */ throw new Error( `S3 ${action} failed (${code}${status ? ` ${status}` : ''}): ${error.message}`, @@ -165,7 +246,6 @@ export const s3Destination: DrainDestination | null = null return { async deliver({ body, contentType, metadata, signal }) { - if (endpointCheck === null) endpointCheck = assertEndpointIsPublic(config.endpoint) - await endpointCheck const key = buildObjectKey(config.prefix, metadata) const keyBytes = Buffer.byteLength(key, 'utf8') if (keyBytes > MAX_S3_KEY_BYTES) { diff --git a/apps/sim/lib/data-drains/destinations/snowflake.test.ts b/apps/sim/lib/data-drains/destinations/snowflake.test.ts index b0f8b4cf2c9..55ae9d1f1fc 100644 --- a/apps/sim/lib/data-drains/destinations/snowflake.test.ts +++ b/apps/sim/lib/data-drains/destinations/snowflake.test.ts @@ -62,8 +62,9 @@ describe('snowflakeDestination', () => { const payload = JSON.parse(init.body as string) expect(payload.statement).toContain('INSERT INTO "DB"."PUBLIC"."DRAINS"') - expect(payload.statement).toContain('VALUES (PARSE_JSON(?)), (PARSE_JSON(?))') - expect(payload.statement.match(/PARSE_JSON\(\?\)/g)).toHaveLength(2) + expect(payload.statement).toBe( + 'INSERT INTO "DB"."PUBLIC"."DRAINS" ("DATA") SELECT PARSE_JSON(column1) FROM VALUES (?), (?)' + ) expect(payload.bindings['1']).toEqual({ type: 'TEXT', value: JSON.stringify({ id: 'a' }) }) expect(payload.bindings['2']).toEqual({ type: 'TEXT', value: JSON.stringify({ id: 'b' }) }) expect(payload.warehouse).toBe('WH') diff --git a/apps/sim/lib/data-drains/destinations/snowflake.ts b/apps/sim/lib/data-drains/destinations/snowflake.ts index ab43eeac1e1..b5550495bec 100644 --- a/apps/sim/lib/data-drains/destinations/snowflake.ts +++ b/apps/sim/lib/data-drains/destinations/snowflake.ts @@ -193,8 +193,8 @@ function quoteIdentifier(name: string): string { function buildStatement(config: SnowflakeDestinationConfig, rowCount: number): string { const column = quoteIdentifier(config.column ?? 'DATA') const target = `${quoteIdentifier(config.database)}.${quoteIdentifier(config.schema)}.${quoteIdentifier(config.table)}` - const placeholders = Array.from({ length: rowCount }, () => '(PARSE_JSON(?))').join(', ') - return `INSERT INTO ${target} (${column}) VALUES ${placeholders}` + const placeholders = Array.from({ length: rowCount }, () => '(?)').join(', ') + return `INSERT INTO ${target} (${column}) SELECT PARSE_JSON(column1) FROM VALUES ${placeholders}` } function isRetryableStatus(status: number): boolean { @@ -261,7 +261,6 @@ async function executeStatement(input: ExecuteInput): Promise { lastError = error logger.warn('Snowflake request failed', { attempt, - error: toError(error).message, }) if (input.signal.aborted || attempt === EXECUTE_MAX_ATTEMPTS) throw error await interruptibleSleep( @@ -367,7 +366,6 @@ async function pollStatement(input: PollInput): Promise { logger.warn('Snowflake poll request failed, retrying', { attempt: retryAttempt, delayMs: delay, - error: toError(error).message, }) await interruptibleSleep(delay, input.signal) skipIntervalSleep = true diff --git a/apps/sim/lib/data-drains/destinations/webhook.ts b/apps/sim/lib/data-drains/destinations/webhook.ts index 0a61d4a4114..70f9ca70a42 100644 --- a/apps/sim/lib/data-drains/destinations/webhook.ts +++ b/apps/sim/lib/data-drains/destinations/webhook.ts @@ -1,6 +1,5 @@ import { createHmac } from 'node:crypto' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { z } from 'zod' @@ -205,9 +204,8 @@ export const webhookDestination: DrainDestination< } catch (error) { lastError = error logger.debug('Webhook delivery attempt failed', { - url: config.url, + host: new URL(config.url).hostname, attempt, - error: toError(error).message, }) } if (response) { @@ -217,7 +215,7 @@ export const webhookDestination: DrainDestination< response.headers.get('x-amzn-trace-id') ?? null logger.debug('Webhook chunk delivered', { - url: config.url, + host: new URL(config.url).hostname, attempt, status: response.status, bytes: body.byteLength, diff --git a/apps/sim/lib/data-drains/dispatcher.test.ts b/apps/sim/lib/data-drains/dispatcher.test.ts deleted file mode 100644 index d42e80b53b1..00000000000 --- a/apps/sim/lib/data-drains/dispatcher.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { dbChainMockFns, resetDbChainMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing' -import { asyncJobsMock, asyncJobsMockFns } from '@sim/testing/mocks/async-jobs.mock' -import { - billingSubscriptionMock, - billingSubscriptionMockFns, -} from '@sim/testing/mocks/billing-subscription.mock' -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/billing/core/subscription', () => billingSubscriptionMock) -vi.mock('@/lib/core/async-jobs', () => asyncJobsMock) - -import { dispatchDueDrains } from '@/lib/data-drains/dispatcher' - -const mockEnqueue = asyncJobsMockFns.mockJobQueue.enqueue -mockEnqueue.mockResolvedValue('job-id') -const mockIsEnterprise = billingSubscriptionMockFns.mockIsOrganizationOnEnterprisePlan - -function mockCandidates(rows: Array<{ id: string; organizationId: string }>) { - // db.select().from().where() — override `from` so awaiting `.where(pred)` - // resolves with the candidate rows. - dbChainMockFns.from.mockReturnValueOnce({ - where: vi.fn().mockResolvedValueOnce(rows), - } as never) -} - -beforeEach(() => { - resetDbChainMock() -}) - -beforeAll(() => { - setEnvFlags({ isBillingEnabled: true }) -}) - -afterAll(resetEnvFlagsMock) - -describe('dispatchDueDrains', () => { - it('skips drains for orgs not on enterprise plan', async () => { - dbChainMockFns.returning.mockResolvedValueOnce([]) // reaper - mockCandidates([{ id: 'd1', organizationId: 'org-a' }]) - mockIsEnterprise.mockResolvedValueOnce(false) - - const result = await dispatchDueDrains() - expect(result).toMatchObject({ candidates: 1, dispatched: 0, skipped: 1 }) - expect(mockEnqueue).not.toHaveBeenCalled() - }) - - it('does not enqueue when claim loses the race', async () => { - dbChainMockFns.returning - .mockResolvedValueOnce([]) // reaper - .mockResolvedValueOnce([]) // claim returns nothing — lost the race - mockCandidates([{ id: 'd1', organizationId: 'org-a' }]) - mockIsEnterprise.mockResolvedValueOnce(true) - - const result = await dispatchDueDrains() - expect(result.dispatched).toBe(0) - expect(mockEnqueue).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/data-drains/dispatcher.ts b/apps/sim/lib/data-drains/dispatcher.ts index aec56a3bf3a..761c3e7882d 100644 --- a/apps/sim/lib/data-drains/dispatcher.ts +++ b/apps/sim/lib/data-drains/dispatcher.ts @@ -2,23 +2,21 @@ import { db } from '@sim/db' import { dataDrainRuns, dataDrains } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { and, eq, isNull, lt, or } from 'drizzle-orm' +import { and, asc, eq, gt, isNull, lt, or, sql } from 'drizzle-orm' import { isOrganizationOnEnterprisePlan } from '@/lib/billing/core/subscription' -import { getJobQueue } from '@/lib/core/async-jobs' import { isBillingEnabled } from '@/lib/core/config/env-flags' +import { enqueueDrain } from '@/lib/data-drains/enqueue' const logger = createLogger('DataDrainsDispatcher') const HOUR_MS = 60 * 60 * 1000 const DAY_MS = 24 * HOUR_MS +const DISPATCH_PAGE_SIZE = 200 /** - * Cron fires hourly. Without a buffer, a drain that finishes a few minutes - * after the tick (lastRunAt = 10:05) won't satisfy `lastRunAt < now - cadence` - * at the next tick (10:05 < 10:00 is false), so an "hourly" drain effectively - * runs every two hours. Subtracting a small buffer from the cadence absorbs - * normal run duration plus cron jitter without allowing back-to-back runs - * within the same tick. + * Cron start times drift. A small buffer lets the next hourly tick claim a + * drain whose preceding tick started late. Cadence is measured from the + * worker's claim timestamp, independently of delivery duration. */ const CADENCE_BUFFER_MS = 5 * 60 * 1000 @@ -34,23 +32,27 @@ const ORPHAN_THRESHOLD_MS = 60 * 60 * 1000 * Marks `running` rows older than the orphan threshold as `failed`. Without * this, a worker crash leaves run history permanently misleading and (worse) * the drain row's `lastRunAt` reflects a successful claim that never finished - * — but the drain `cursor` never advanced, so re-running is safe. + * — the next run resumes after the last acknowledged checkpoint. */ export async function reapOrphanedRuns(now: Date = new Date()): Promise<{ reaped: number }> { const cutoff = new Date(now.getTime() - ORPHAN_THRESHOLD_MS) - const reaped = await db - .update(dataDrainRuns) - .set({ - status: 'failed', - finishedAt: now, - error: `Orphaned run reaped after exceeding ${ORPHAN_THRESHOLD_MS / 60_000}m without completion`, - }) - .where(and(eq(dataDrainRuns.status, 'running'), lt(dataDrainRuns.startedAt, cutoff))) - .returning({ id: dataDrainRuns.id }) - if (reaped.length > 0) { - logger.warn('Reaped orphaned data drain runs', { count: reaped.length }) + const result = await db.execute<{ count: number }>(sql` + WITH reaped AS ( + UPDATE ${dataDrainRuns} + SET status = 'failed', + finished_at = ${sql.param(now, dataDrainRuns.finishedAt)}, + error = ${`Orphaned run reaped after exceeding ${ORPHAN_THRESHOLD_MS / 60_000}m without completion`} + WHERE ${dataDrainRuns.status} = 'running' + AND ${dataDrainRuns.startedAt} < ${sql.param(cutoff, dataDrainRuns.startedAt)} + RETURNING 1 + ) + SELECT count(*)::integer AS count FROM reaped + `) + const reaped = result[0]?.count ?? 0 + if (reaped > 0) { + logger.warn('Reaped orphaned data drain runs', { count: reaped }) } - return { reaped: reaped.length } + return { reaped } } /** @@ -81,19 +83,6 @@ export async function dispatchDueDrains(now: Date = new Date()): Promise<{ ) ) - const candidates = await db - .select({ - id: dataDrains.id, - organizationId: dataDrains.organizationId, - lastRunAt: dataDrains.lastRunAt, - }) - .from(dataDrains) - .where(duePredicate) - - if (candidates.length === 0) { - return { candidates: 0, dispatched: 0, skipped: 0, reaped } - } - // Self-hosted deployments have no subscription infra; `DATA_DRAINS_ENABLED` // is the global on/off there. Cache per-org so a multi-drain org pays one // billing lookup. @@ -107,80 +96,95 @@ export async function dispatchDueDrains(now: Date = new Date()): Promise<{ return result } - const queue = await getJobQueue() let dispatched = 0 let skipped = 0 - - for (const candidate of candidates) { - let enterprise: boolean - try { - enterprise = await isEnterprise(candidate.organizationId) - } catch (error) { - // A billing-API failure for one org must not abort the whole batch — - // skip this drain and let the next cron tick retry it. - logger.warn('Enterprise check failed; skipping drain', { - drainId: candidate.id, - organizationId: candidate.organizationId, - error, + let candidates = 0 + let afterId: string | undefined + + while (true) { + const page = await db + .select({ + id: dataDrains.id, + organizationId: dataDrains.organizationId, + lastRunAt: dataDrains.lastRunAt, }) - skipped++ - continue - } - if (!enterprise) { - skipped++ - continue - } - - // Conditional claim — re-asserts the due predicate to lose to any other - // dispatcher or manual-run path that's already moved this drain forward. - const claimed = await db - .update(dataDrains) - .set({ lastRunAt: now, updatedAt: now }) - .where(and(eq(dataDrains.id, candidate.id), duePredicate)) - .returning({ id: dataDrains.id }) - - if (claimed.length === 0) continue - - try { - // concurrencyKey serializes runs of the same drain on the job queue, so - // a manual run-now racing a cron claim can never execute in parallel. - await queue.enqueue( - 'run-data-drain', - { drainId: candidate.id, trigger: 'cron' }, - { concurrencyKey: `data-drain:${candidate.id}` } - ) - dispatched++ - } catch (error) { - // Roll back the claim so a transient queue outage doesn't delay this - // drain by a full cadence. Scoped to our own claim timestamp so it - // can't trample a concurrent advance. The rollback itself is guarded - // so a DB error here doesn't abort the rest of the batch. + .from(dataDrains) + .where(and(duePredicate, afterId ? gt(dataDrains.id, afterId) : undefined)) + .orderBy(asc(dataDrains.id)) + .limit(DISPATCH_PAGE_SIZE) + if (page.length === 0) break + candidates += page.length + enterpriseCache.clear() + + for (const candidate of page) { + let enterprise: boolean try { - await db - .update(dataDrains) - .set({ lastRunAt: candidate.lastRunAt, updatedAt: now }) - .where(and(eq(dataDrains.id, candidate.id), eq(dataDrains.lastRunAt, now))) - } catch (rollbackError) { - logger.error('Failed to roll back data-drain claim after enqueue failure', { + enterprise = await isEnterprise(candidate.organizationId) + } catch (error) { + // A billing-API failure for one org must not abort the whole batch — + // skip this drain and let the next cron tick retry it. + logger.warn('Enterprise check failed; skipping drain', { drainId: candidate.id, - enqueueError: toError(error).message, - rollbackError: toError(rollbackError).message, + organizationId: candidate.organizationId, + error, }) + skipped++ continue } - logger.error('Failed to enqueue data-drain job; rolled back claim', { - drainId: candidate.id, - error, - }) + if (!enterprise) { + skipped++ + continue + } + + // Conditional claim — re-asserts the due predicate to lose to any other + // dispatcher or manual-run path that's already moved this drain forward. + const claimed = await db + .update(dataDrains) + .set({ lastRunAt: now, updatedAt: now }) + .where(and(eq(dataDrains.id, candidate.id), duePredicate)) + .returning({ id: dataDrains.id }) + + if (claimed.length === 0) continue + + try { + // Queue limits bound execution; the database claim also fences workers + // across app processes and concurrent manual requests. + await enqueueDrain(candidate.id, 'cron') + dispatched++ + } catch (error) { + // Roll back the claim so a transient queue outage doesn't delay this + // drain by a full cadence. Scoped to our own claim timestamp so it + // can't trample a concurrent advance. The rollback itself is guarded + // so a DB error here doesn't abort the rest of the batch. + try { + await db + .update(dataDrains) + .set({ lastRunAt: candidate.lastRunAt, updatedAt: now }) + .where(and(eq(dataDrains.id, candidate.id), eq(dataDrains.lastRunAt, now))) + } catch (rollbackError) { + logger.error('Failed to roll back data-drain claim after enqueue failure', { + drainId: candidate.id, + enqueueError: toError(error).message, + rollbackError: toError(rollbackError).message, + }) + continue + } + logger.error('Failed to enqueue data-drain job; rolled back claim', { + drainId: candidate.id, + error, + }) + } } + afterId = page[page.length - 1].id + if (page.length < DISPATCH_PAGE_SIZE) break } logger.info('Data drain dispatch complete', { - candidates: candidates.length, + candidates, dispatched, skipped, reaped, }) - return { candidates: candidates.length, dispatched, skipped, reaped } + return { candidates, dispatched, skipped, reaped } } diff --git a/apps/sim/lib/data-drains/enqueue.ts b/apps/sim/lib/data-drains/enqueue.ts new file mode 100644 index 00000000000..dc6aee9c73a --- /dev/null +++ b/apps/sim/lib/data-drains/enqueue.ts @@ -0,0 +1,31 @@ +import { getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' +import { runDrain } from '@/lib/data-drains/service' +import type { RunTrigger } from '@/lib/data-drains/types' + +const INLINE_CONCURRENCY_LIMIT = 2 + +/** Runs one bounded export window and schedules a serialized continuation when a backlog remains. */ +export async function runQueuedDrain(drainId: string, trigger: RunTrigger, signal: AbortSignal) { + const result = await runDrain(drainId, trigger, { signal }) + if (result.status === 'success' && result.hasMore) { + await enqueueDrain(drainId, trigger) + } + return result +} + +/** Bounds inline exports per app process and uses per-drain serialization on isolated external workers. */ +export async function enqueueDrain(drainId: string, trigger: RunTrigger): Promise { + const queue = await getJobQueue() + const inline = shouldExecuteInline() + return queue.enqueue( + 'run-data-drain', + { drainId, trigger }, + { + concurrencyKey: inline ? 'data-drains:inline' : `data-drain:${drainId}`, + concurrencyLimit: inline ? INLINE_CONCURRENCY_LIMIT : 1, + maxDurationSeconds: DATA_DRAIN_LIMITS.hardDurationMs / 1000, + runner: async (_payload, signal) => runQueuedDrain(drainId, trigger, signal), + } + ) +} diff --git a/apps/sim/lib/data-drains/errors.ts b/apps/sim/lib/data-drains/errors.ts new file mode 100644 index 00000000000..7d10012c970 --- /dev/null +++ b/apps/sim/lib/data-drains/errors.ts @@ -0,0 +1,64 @@ +import { toError } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { truncate } from '@sim/utils/string' +import { redactExactSensitiveValues } from '@/lib/core/security/redaction' + +const MAX_CREDENTIAL_NODES = 256 +const MAX_CREDENTIAL_DEPTH = 8 +const MAX_CREDENTIAL_BYTES = 1024 * 1024 +const MAX_ERROR_INPUT_CHARS = 16_000 +const MAX_ERROR_OUTPUT_CHARS = 4000 +const REDACTED_ERROR = 'Data drain destination failed (details redacted)' + +/** + * Removes saved credential values, including leaves inside JSON credentials, before errors reach + * history, audit, logs or the job queue. Malformed JSON and collection limits hide the entire + * diagnostic because providers may echo only a fragment of a credential container. + */ +export function createCredentialErrorRedactor(credentials?: unknown): (error: unknown) => string { + const secrets: string[] = [] + let nodes = 0 + let bytes = 0 + let complete = credentials !== undefined + + function collect(value: unknown, depth: number): void { + if (++nodes > MAX_CREDENTIAL_NODES || depth > MAX_CREDENTIAL_DEPTH) { + complete = false + return + } + if (typeof value === 'string') { + bytes += Buffer.byteLength(value, 'utf8') + if (bytes > MAX_CREDENTIAL_BYTES) { + complete = false + return + } + if (value) secrets.push(value) + const trimmed = value.trimStart() + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + collect(JSON.parse(value), depth + 1) + } catch { + complete = false + } + } + } else if (Array.isArray(value)) { + for (const item of value) { + if (!complete) break + collect(item, depth + 1) + } + } else if (isRecordLike(value)) { + for (const key in value) { + if (!complete) break + if (Object.hasOwn(value, key)) collect(value[key], depth + 1) + } + } + } + + collect(credentials, 0) + return (error) => { + if (!complete) return REDACTED_ERROR + const message = toError(error).message + if (message.length > MAX_ERROR_INPUT_CHARS) return REDACTED_ERROR + return truncate(redactExactSensitiveValues(message, secrets), MAX_ERROR_OUTPUT_CHARS, '') + } +} diff --git a/apps/sim/lib/data-drains/limits.ts b/apps/sim/lib/data-drains/limits.ts new file mode 100644 index 00000000000..e4e0d4610a4 --- /dev/null +++ b/apps/sim/lib/data-drains/limits.ts @@ -0,0 +1,10 @@ +export const DATA_DRAIN_LIMITS = { + pageRows: 100, + maxRowBytes: 1024 * 1024, + maxChunkBytes: 4 * 1024 * 1024, + maxChunksPerRun: 100, + maxRowsPerRun: 10_000, + maxBytesPerRun: 100 * 1024 * 1024, + softDurationMs: 20 * 60_000, + hardDurationMs: 25 * 60_000, +} as const diff --git a/apps/sim/lib/data-drains/service.integration.ts b/apps/sim/lib/data-drains/service.integration.ts new file mode 100644 index 00000000000..cdcaa82bdd9 --- /dev/null +++ b/apps/sim/lib/data-drains/service.integration.ts @@ -0,0 +1,531 @@ +import { mkdir, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { db } from '@sim/db' +import { + asyncJobs, + auditLog, + dataDrainRuns, + dataDrains, + member, + organization, + user, +} from '@sim/db/schema' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { createDeferred } from '@sim/testing/helpers/deferred' +import { + billingSubscriptionMock, + billingSubscriptionMockFns, +} from '@sim/testing/mocks/billing-subscription.mock' +import { envFlagsMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' +import { sleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, sql } from 'drizzle-orm' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetDestination } = vi.hoisted(() => ({ mockGetDestination: vi.fn() })) +vi.mock('@/lib/data-drains/destinations/registry', () => ({ getDestination: mockGetDestination })) +vi.mock('@/lib/billing/core/subscription', () => billingSubscriptionMock) +vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) + +import { testDataDrain } from '@/lib/data-drains/application/use-cases' +import { dispatchDueDrains } from '@/lib/data-drains/dispatcher' +import { encryptCredentials } from '@/lib/data-drains/encryption' +import { enqueueDrain } from '@/lib/data-drains/enqueue' +import { runDrain } from '@/lib/data-drains/service' + +const ids = { user: generateId(), organization: generateId() } +const checks: Array<{ name: string; status: string; durationMs: number }> = [] +let drainId = '' +let fixtureDrainIds: string[] = [] +let extraOrganizationIds: string[] = [] +let testStartedAt = 0 +let received: Array> = [] + +async function seedRows(count: number, description?: string) { + const createdAt = new Date(Date.now() - 10 * 60_000) + await db.insert(auditLog).values( + Array.from({ length: count }, (_, index) => ({ + id: `${drainId}-${index.toString().padStart(6, '0')}`, + action: 'fixture.updated', + resourceType: 'fixture', + metadata: { organizationId: ids.organization }, + description, + createdAt, + })) + ) +} + +async function drainRow() { + const [row] = await db.select().from(dataDrains).where(eq(dataDrains.id, drainId)) + if (!row) throw new Error('Drain fixture disappeared') + return row +} + +function destination(deliver?: (rows: Array<{ id: string }>) => Promise, close?: () => void) { + return { + configSchema: { parse: (value: unknown) => value }, + credentialsSchema: { parse: (value: unknown) => value }, + openSession: () => ({ + async deliver({ body }: { body: Buffer }) { + const rows = body + .toString('utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + await deliver?.(rows) + received.push(rows) + return { locator: `fixture://${drainId}/${received.length}` } + }, + async close() { + close?.() + }, + }), + } +} + +beforeAll(async () => { + await db.insert(user).values({ + id: ids.user, + name: 'Drain integrity fixture', + email: `${ids.user}@drain.test`, + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + }) + await db.insert(organization).values({ + id: ids.organization, + name: 'Drain integrity fixture', + slug: ids.organization, + createdAt: new Date(), + }) + await db.insert(member).values({ + id: generateId(), + organizationId: ids.organization, + userId: ids.user, + role: 'admin', + createdAt: new Date(), + }) +}) + +beforeEach(async () => { + testStartedAt = Date.now() + drainId = generateId() + fixtureDrainIds = [drainId] + extraOrganizationIds = [] + received = [] + resetEnvFlagsMock() + setEnvFlags({ isDataDrainsEnabled: true }) + billingSubscriptionMockFns.mockIsOrganizationOnEnterprisePlan.mockResolvedValue(true) + mockGetDestination.mockReturnValue(destination()) + await db.insert(dataDrains).values({ + id: drainId, + organizationId: ids.organization, + name: drainId, + source: 'audit_logs', + destinationType: 'webhook', + destinationConfig: {}, + destinationCredentials: await encryptCredentials({}), + scheduleCadence: 'hourly', + enabled: true, + createdBy: ids.user, + }) +}) + +afterEach(async (context) => { + checks.push({ + name: context.task.name, + status: context.task.result?.state ?? 'unknown', + durationMs: Date.now() - testStartedAt, + }) + await db.delete(dataDrains).where(inArray(dataDrains.id, fixtureDrainIds)) + await db + .delete(auditLog) + .where( + inArray(sql`${auditLog.metadata}->>'organizationId'`, [ + ids.organization, + ...extraOrganizationIds, + ]) + ) + if (extraOrganizationIds.length > 0) { + await db.delete(organization).where(inArray(organization.id, extraOrganizationIds)) + } +}) + +afterAll(async () => { + await db.delete(organization).where(eq(organization.id, ids.organization)) + await db.delete(user).where(eq(user.id, ids.user)) + const reportPath = + process.env.DATA_DRAINS_REPORT_PATH ?? 'test-results/data-drains-integration.json' + await mkdir(path.dirname(reportPath), { recursive: true }) + await writeFile(reportPath, JSON.stringify({ checks }, null, 2)) +}) + +describe('data drain durable delivery', () => { + it('fails closed when malformed JSON credentials expose only a private-key fragment', async () => { + await seedRows(1) + const fragment = 'malformed-private-key-fragment-028f' + await db + .update(dataDrains) + .set({ + destinationCredentials: await encryptCredentials({ + serviceAccountJson: `{"private_key":"${fragment}...`, + }), + }) + .where(eq(dataDrains.id, drainId)) + mockGetDestination.mockReturnValue( + destination(async () => { + throw new Error(`Invalid JSON near '${fragment}'`) + }) + ) + const failure = await runDrain(drainId, 'cron').catch((error: unknown) => error) + expect(failure).toBeInstanceOf(Error) + if (!(failure instanceof Error)) throw new Error('Expected delivery failure') + expect(failure.message).not.toContain(fragment) + const [run] = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(run.status).toBe('failed') + expect(run.error).not.toContain(fragment) + }) + + it('redacts saved credential values from persisted run failures and propagated errors', async () => { + await seedRows(1) + const bearerToken = 'drain-fixture-bearer-6a81' + const privateKey = 'drain-fixture-private-key-94d2' + await db + .update(dataDrains) + .set({ + destinationCredentials: await encryptCredentials({ + bearerToken, + serviceAccountJson: JSON.stringify({ private_key: privateKey }), + }), + }) + .where(eq(dataDrains.id, drainId)) + mockGetDestination.mockReturnValue( + destination(async () => { + throw new Error(`HTTP 403 rejected credentials ${bearerToken} and ${privateKey}`) + }) + ) + const failure = await runDrain(drainId, 'cron').catch((error: unknown) => error) + expect(failure).toBeInstanceOf(Error) + if (!(failure instanceof Error)) throw new Error('Expected delivery failure') + expect(failure.message).toContain('HTTP 403') + expect(failure.message).not.toContain(bearerToken) + expect(failure.message).not.toContain(privateKey) + const [run] = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(run.status).toBe('failed') + expect(run.error).toContain('HTTP 403') + expect(run.error).not.toContain(bearerToken) + expect(run.error).not.toContain(privateKey) + expect(run.error?.length).toBeLessThanOrEqual(4000) + }) + + it('redacts connection-test credential echoes from its response and organization audit', async () => { + const bearerToken = 'connection-fixture-bearer-517d' + const privateKey = 'connection-fixture-private-key-b719' + await db + .update(dataDrains) + .set({ + destinationCredentials: await encryptCredentials({ + bearerToken, + serviceAccountJson: JSON.stringify({ private_key: privateKey }), + }), + }) + .where(eq(dataDrains.id, drainId)) + mockGetDestination.mockReturnValue({ + ...destination(), + async test() { + throw new Error(`HTTP 401 rejected credentials ${bearerToken} and ${privateKey}`) + }, + }) + const result = await testDataDrain.execute({ + principal: createSessionPrincipal({ userId: ids.user }), + input: { organizationId: ids.organization, drainId }, + }) + expect(result.ok).toBe(false) + const audit = await vi.waitFor(async () => { + const [entry] = await db + .select() + .from(auditLog) + .where(and(eq(auditLog.resourceId, drainId), eq(auditLog.action, 'data_drain.tested'))) + expect(entry).toBeDefined() + return entry + }) + const persistedAndReturned = JSON.stringify({ result, metadata: audit.metadata }) + expect(persistedAndReturned).toContain('HTTP 401') + expect(persistedAndReturned).not.toContain(bearerToken) + expect(persistedAndReturned).not.toContain(privateKey) + }) + + it('admits one worker for the same drain while delivery is in flight', async () => { + await seedRows(1) + const entered = createDeferred() + const release = createDeferred() + let firstDelivery = true + mockGetDestination.mockReturnValue( + destination(async () => { + if (!firstDelivery) return + firstDelivery = false + entered.resolve() + await release.promise + }) + ) + const first = runDrain(drainId, 'cron') + await entered.promise + try { + const second = await runDrain(drainId, 'manual') + expect(second.status).toBe('skipped') + const runs = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(runs).toHaveLength(1) + } finally { + release.resolve() + await first + } + expect(received.flat()).toHaveLength(1) + }) + + it('fences a worker whose running history has been reaped', async () => { + await seedRows(1) + const entered = createDeferred() + const release = createDeferred() + mockGetDestination.mockReturnValue( + destination(async () => { + entered.resolve() + await release.promise + }) + ) + const pending = runDrain(drainId, 'cron') + await entered.promise + await db + .update(dataDrainRuns) + .set({ status: 'failed', error: 'reaped' }) + .where(and(eq(dataDrainRuns.drainId, drainId), eq(dataDrainRuns.status, 'running'))) + release.resolve() + await expect(pending).rejects.toThrow(/lease/i) + expect((await drainRow()).cursor).toBeNull() + const [run] = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(run.status).toBe('failed') + expect(run.error).toBe('reaped') + }) + + it('keeps acknowledged progress when a later delivery fails', async () => { + await seedRows(1001) + let attempts = 0 + mockGetDestination.mockReturnValue( + destination(async () => { + if (++attempts === 2) throw new Error('destination unavailable') + }) + ) + await expect(runDrain(drainId, 'cron')).rejects.toThrow('destination unavailable') + const confirmedRows = received.flat() + const confirmedLast = confirmedRows[confirmedRows.length - 1] + expect(confirmedRows.length).toBeGreaterThan(0) + expect(JSON.parse((await drainRow()).cursor ?? '{}').id).toBe(confirmedLast.id) + const [run] = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(run.status).toBe('failed') + expect(run.rowsExported).toBe(confirmedRows.length) + expect(run.cursorAfter).toBe((await drainRow()).cursor) + }) + + it('stops a backlog at a bounded window and resumes without losing rows', async () => { + await seedRows(10001) + const first = await runDrain(drainId, 'cron') + expect(first.rowsExported).toBeGreaterThan(0) + expect(first.rowsExported).toBeLessThanOrEqual(10000) + expect(first.locators.length).toBeLessThanOrEqual(100) + for (let attempts = 0; attempts < 10 && received.flat().length < 10001; attempts++) { + await runDrain(drainId, 'cron') + } + const deliveredIds = received.flat().map((row) => row.id) + expect(deliveredIds).toHaveLength(10001) + expect(new Set(deliveredIds).size).toBe(10001) + }, 30_000) + + it('rejects an oversized record before destination delivery', async () => { + await seedRows(1, 'x'.repeat(1024 * 1024 + 1)) + await expect(runDrain(drainId, 'cron')).rejects.toThrow(/record.*bytes|row.*bytes/i) + expect(received).toHaveLength(0) + expect((await drainRow()).cursor).toBeNull() + }) + + it('executes a dispatched drain through the database queue fallback', async () => { + await seedRows(1) + await dispatchDueDrains() + await vi.waitFor( + async () => { + expect((await drainRow()).cursor).not.toBeNull() + const [run] = await db + .select() + .from(dataDrainRuns) + .where(eq(dataDrainRuns.drainId, drainId)) + expect(run.status).toBe('success') + }, + { timeout: 3000 } + ) + expect(received.flat()).toHaveLength(1) + }) + + it('marks the queued job failed when destination delivery fails', async () => { + await seedRows(1) + mockGetDestination.mockReturnValue( + destination(async () => { + throw new Error('destination unavailable') + }) + ) + const jobId = await enqueueDrain(drainId, 'cron') + await vi.waitFor(async () => { + const [job] = await db.select().from(asyncJobs).where(eq(asyncJobs.id, jobId)) + expect(job.status).toBe('failed') + expect(job.error).toContain('destination unavailable') + const [run] = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(run.status).toBe('failed') + }) + expect((await drainRow()).cursor).toBeNull() + expect(received).toHaveLength(0) + }) + + it('claims a due drain once when dispatchers race', async () => { + await seedRows(1) + const now = new Date() + const dispatches = await Promise.all([dispatchDueDrains(now), dispatchDueDrains(now)]) + expect(dispatches.reduce((total, dispatch) => total + dispatch.dispatched, 0)).toBe(1) + await vi.waitFor(async () => { + const [run] = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(run.status).toBe('success') + }) + expect(received.flat()).toHaveLength(1) + }) + + it('bounds the database fallback to two active drains across organizations', async () => { + await seedRows(1) + const original = await drainRow() + const extraIds = [generateId(), generateId()] + extraOrganizationIds = [generateId(), generateId()] + fixtureDrainIds.push(...extraIds) + await db.insert(organization).values( + extraOrganizationIds.map((id) => ({ + id, + name: id, + slug: id, + createdAt: new Date(), + })) + ) + await db.insert(dataDrains).values( + extraIds.map((id, index) => ({ + ...original, + id, + name: id, + organizationId: extraOrganizationIds[index], + })) + ) + await db.insert(auditLog).values( + extraOrganizationIds.map((organizationId) => ({ + id: generateId(), + action: 'fixture.updated', + resourceType: 'fixture', + metadata: { organizationId }, + createdAt: new Date(Date.now() - 10 * 60_000), + })) + ) + const firstTwoEntered = createDeferred() + const thirdEntered = createDeferred() + const release = createDeferred() + let active = 0 + let entered = 0 + let peakActive = 0 + mockGetDestination.mockReturnValue( + destination(async () => { + entered++ + active++ + peakActive = Math.max(peakActive, active) + if (entered === 2) firstTwoEntered.resolve() + if (entered === 3) thirdEntered.resolve() + await release.promise + active-- + }) + ) + const jobs = await Promise.all([ + enqueueDrain(drainId, 'cron'), + enqueueDrain(extraIds[0], 'cron'), + ]) + await firstTwoEntered.promise + jobs.push(await enqueueDrain(extraIds[1], 'cron')) + try { + const observation = await Promise.race([ + thirdEntered.promise.then(() => 'entered'), + sleep(300).then(() => 'queued'), + ]) + expect(observation).toBe('queued') + expect(peakActive).toBe(2) + } finally { + release.resolve() + await vi.waitFor(async () => { + const rows = await db.select().from(asyncJobs).where(inArray(asyncJobs.id, jobs)) + expect(rows).toHaveLength(3) + expect(rows.every((row) => row.status === 'completed')).toBe(true) + }) + } + expect(received.flat()).toHaveLength(3) + }) + + it('continues a queued backlog until every bounded window is delivered', async () => { + await seedRows(10001) + await dispatchDueDrains() + await vi.waitFor( + async () => { + expect(received.flat()).toHaveLength(10001) + const runs = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(runs.length).toBeGreaterThan(1) + expect(runs.every((run) => run.status === 'success')).toBe(true) + }, + { timeout: 10_000 } + ) + expect(new Set(received.flat().map((row) => row.id)).size).toBe(10001) + }, 30_000) + + it('rechecks an organization entitlement before a queued worker reads or delivers its data', async () => { + await seedRows(1) + setEnvFlags({ isBillingEnabled: true }) + billingSubscriptionMockFns.mockIsOrganizationOnEnterprisePlan.mockResolvedValue(false) + const result = await runDrain(drainId, 'cron') + expect(result.status).toBe('skipped') + expect(received).toHaveLength(0) + expect((await drainRow()).cursor).toBeNull() + expect( + await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + ).toHaveLength(0) + }) + + it('closes its session and keeps the acknowledged checkpoint when cancellation arrives after delivery', async () => { + await seedRows(1) + const controller = new AbortController() + let closed = false + mockGetDestination.mockReturnValue( + destination( + async () => controller.abort(), + () => { + closed = true + } + ) + ) + await expect(runDrain(drainId, 'cron', { signal: controller.signal })).rejects.toThrow( + /cancelled|aborted/i + ) + expect(closed).toBe(true) + expect(received.flat()).toHaveLength(1) + expect(JSON.parse((await drainRow()).cursor ?? '{}').id).toBe(received[0][0].id) + const [run] = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + expect(run.status).toBe('failed') + }) + + it('preserves successful delivery when releasing the destination session fails', async () => { + await seedRows(1) + mockGetDestination.mockReturnValue( + destination(undefined, () => { + throw new Error('close failed') + }) + ) + const result = await runDrain(drainId, 'cron') + expect(result.status).toBe('success') + expect(received.flat()).toHaveLength(1) + expect((await drainRow()).cursor).toBe(result.cursorAfter) + }) +}) diff --git a/apps/sim/lib/data-drains/service.test.ts b/apps/sim/lib/data-drains/service.test.ts deleted file mode 100644 index 411084c2adb..00000000000 --- a/apps/sim/lib/data-drains/service.test.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockGetSource, mockGetDestination, mockDecryptCredentials } = vi.hoisted(() => ({ - mockGetSource: vi.fn(), - mockGetDestination: vi.fn(), - mockDecryptCredentials: vi.fn(), -})) - -vi.mock('@/lib/data-drains/sources/registry', () => ({ getSource: mockGetSource })) -vi.mock('@/lib/data-drains/destinations/registry', () => ({ getDestination: mockGetDestination })) -vi.mock('@/lib/data-drains/encryption', () => ({ decryptCredentials: mockDecryptCredentials })) - -import { runDrain } from '@/lib/data-drains/service' - -type Row = { id: string; ts: string } - -function makeSource(pages: Row[][]) { - return { - type: 'workflow_logs' as const, - displayName: 'Test', - pages: vi.fn(async function* () { - for (const page of pages) yield page - }), - serialize: vi.fn((row: Row) => row), - cursorAfter: vi.fn((row: Row) => JSON.stringify({ ts: row.ts, id: row.id })), - } -} - -function makeDestination( - opts: { deliver?: ReturnType; close?: ReturnType } = {} -) { - const deliver = - opts.deliver ?? - vi.fn(async ({ metadata }: { metadata: { sequence: number } }) => ({ - locator: `loc-${metadata.sequence}`, - })) - const close = opts.close ?? vi.fn(async () => {}) - return { - type: 's3' as const, - displayName: 'Test', - configSchema: { parse: (v: unknown) => v }, - credentialsSchema: { parse: (v: unknown) => v }, - openSession: vi.fn(() => ({ deliver, close })), - _deliver: deliver, - _close: close, - } -} - -const baseDrain = { - id: 'drain-1', - organizationId: 'org-1', - enabled: true, - source: 'workflow_logs', - destinationType: 's3', - destinationConfig: {}, - destinationCredentials: 'enc:blob', - cursor: null, -} - -beforeEach(() => { - resetDbChainMock() - mockDecryptCredentials.mockResolvedValue({}) -}) - -describe('runDrain', () => { - it('delivers each page and advances cursor on success', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([baseDrain]) - const source = makeSource([ - [ - { id: 'r1', ts: '2026-01-01T00:00:00.000Z' }, - { id: 'r2', ts: '2026-01-01T00:00:01.000Z' }, - ], - [{ id: 'r3', ts: '2026-01-01T00:00:02.000Z' }], - ]) - const destination = makeDestination() - mockGetSource.mockReturnValue(source) - mockGetDestination.mockReturnValue(destination) - - const result = await runDrain('drain-1', 'cron') - - expect(result.status).toBe('success') - expect(result.rowsExported).toBe(3) - expect(destination._deliver).toHaveBeenCalledTimes(2) - expect(destination._close).toHaveBeenCalledTimes(1) - expect(result.cursorAfter).toBe(JSON.stringify({ ts: '2026-01-01T00:00:02.000Z', id: 'r3' })) - expect(result.locators).toEqual(['loc-0', 'loc-1']) - - // Drain row updated with new cursor; transaction was used. - expect(dbChainMockFns.transaction).toHaveBeenCalled() - const drainUpdate = dbChainMockFns.set.mock.calls.find( - (call) => (call[0] as { cursor?: unknown }).cursor !== undefined - ) - expect(drainUpdate?.[0]).toMatchObject({ cursor: result.cursorAfter }) - }) - - it('does not advance drain cursor when delivery fails', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([{ ...baseDrain, cursor: 'prior' }]) - const source = makeSource([[{ id: 'r1', ts: '2026-01-01T00:00:00.000Z' }]]) - const destination = makeDestination({ - deliver: vi.fn(async () => { - throw new Error('boom') - }), - }) - mockGetSource.mockReturnValue(source) - mockGetDestination.mockReturnValue(destination) - - await expect(runDrain('drain-1', 'cron')).rejects.toThrow('boom') - - // Run row updated with status=failed and cursorAfter equal to prior cursor. - const failedUpdate = dbChainMockFns.set.mock.calls.find( - (call) => (call[0] as { status?: unknown }).status === 'failed' - ) - expect(failedUpdate?.[0]).toMatchObject({ status: 'failed', cursorAfter: 'prior' }) - - // No drain-row update with a new cursor field. - const cursorAdvanced = dbChainMockFns.set.mock.calls.some( - (call) => 'cursor' in (call[0] as object) - ) - expect(cursorAdvanced).toBe(false) - - expect(destination._close).toHaveBeenCalledTimes(1) - }) - - it('closes session even if close throws', async () => { - dbChainMockFns.limit.mockResolvedValueOnce([baseDrain]) - const source = makeSource([]) - const destination = makeDestination({ - close: vi.fn(async () => { - throw new Error('close-failed') - }), - }) - mockGetSource.mockReturnValue(source) - mockGetDestination.mockReturnValue(destination) - - const result = await runDrain('drain-1', 'manual') - expect(result.status).toBe('success') - expect(destination._close).toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/data-drains/service.ts b/apps/sim/lib/data-drains/service.ts index df5709af819..b3ae9db724a 100644 --- a/apps/sim/lib/data-drains/service.ts +++ b/apps/sim/lib/data-drains/service.ts @@ -1,19 +1,20 @@ import { db } from '@sim/db' import { dataDrainRuns, dataDrains } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { eq } from 'drizzle-orm' +import { and, eq } from 'drizzle-orm' +import { isOrganizationOnEnterprisePlan } from '@/lib/billing/core/subscription' +import { isBillingEnabled, isDataDrainsEnabled } from '@/lib/core/config/env-flags' import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { getDestination } from '@/lib/data-drains/destinations/registry' import { decryptCredentials } from '@/lib/data-drains/encryption' +import { createCredentialErrorRedactor } from '@/lib/data-drains/errors' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' import { getSource } from '@/lib/data-drains/sources/registry' import type { Cursor, RunTrigger } from '@/lib/data-drains/types' const logger = createLogger('DataDrainsService') -const CHUNK_SIZE = 1000 - export interface RunDrainResult { drainId: string runId: string @@ -23,26 +24,88 @@ export interface RunDrainResult { cursorBefore: Cursor cursorAfter: Cursor locators: string[] + hasMore: boolean error?: string } /** * Orchestrates one drain export. Source-/destination-agnostic — talks only to - * the registry interfaces. The drain's cursor is advanced only when the entire - * run completes successfully so consumers see at-least-once delivery and can - * dedupe on the per-row `id` field. + * the registry interfaces. Each acknowledged chunk checkpoints its cursor; + * a crash between delivery and checkpoint can replay that chunk, so consumers + * dedupe on the per-row `id` field. A database claim fences concurrent workers. */ export async function runDrain( drainId: string, trigger: RunTrigger, options: { signal?: AbortSignal } = {} ): Promise { - const signal = options.signal ?? new AbortController().signal - const [drain] = await db.select().from(dataDrains).where(eq(dataDrains.id, drainId)).limit(1) - if (!drain) { - throw new Error(`Data drain not found: ${drainId}`) + const startedAt = new Date() + const signal = AbortSignal.any([ + ...(options.signal ? [options.signal] : []), + AbortSignal.timeout(DATA_DRAIN_LIMITS.hardDurationMs), + ]) + signal.throwIfAborted() + const [snapshot] = await db + .select({ + organizationId: dataDrains.organizationId, + enabled: dataDrains.enabled, + cursor: dataDrains.cursor, + }) + .from(dataDrains) + .where(eq(dataDrains.id, drainId)) + .limit(1) + if (!snapshot) throw new Error(`Data drain not found: ${drainId}`) + if ( + !snapshot.enabled || + (!isBillingEnabled && !isDataDrainsEnabled) || + (isBillingEnabled && !(await isOrganizationOnEnterprisePlan(snapshot.organizationId))) + ) { + return { + drainId, + runId: '', + status: 'skipped', + rowsExported: 0, + bytesWritten: 0, + cursorBefore: snapshot.cursor, + cursorAfter: snapshot.cursor, + locators: [], + hasMore: false, + } } - if (!drain.enabled) { + const runId = generateId() + const claim = await db.transaction(async (tx) => { + const [drain] = await tx + .select() + .from(dataDrains) + .where(eq(dataDrains.id, drainId)) + .limit(1) + .for('update') + if (!drain) throw new Error(`Data drain not found: ${drainId}`) + if (!drain.enabled) return { drain, claimed: false } + const [running] = await tx + .select({ id: dataDrainRuns.id }) + .from(dataDrainRuns) + .where(and(eq(dataDrainRuns.drainId, drainId), eq(dataDrainRuns.status, 'running'))) + .limit(1) + if (running) return { drain, claimed: false } + signal.throwIfAborted() + await tx.insert(dataDrainRuns).values({ + id: runId, + drainId, + status: 'running', + trigger, + startedAt, + cursorBefore: drain.cursor, + cursorAfter: drain.cursor, + }) + await tx + .update(dataDrains) + .set({ lastRunAt: startedAt, updatedAt: startedAt }) + .where(eq(dataDrains.id, drainId)) + return { drain, claimed: true } + }) + const { drain } = claim + if (!claim.claimed) { return { drainId, runId: '', @@ -52,28 +115,20 @@ export async function runDrain( cursorBefore: drain.cursor, cursorAfter: drain.cursor, locators: [], + hasMore: false, } } const source = getSource(drain.source) const destination = getDestination(drain.destinationType) - const runId = generateId() - const startedAt = new Date() - await db.insert(dataDrainRuns).values({ - id: runId, - drainId, - status: 'running', - trigger, - startedAt, - cursorBefore: drain.cursor, - }) - const cursorBefore = drain.cursor let cursor: Cursor = drain.cursor + let checkpointCursor: Cursor = drain.cursor let rowsExported = 0 let bytesWritten = 0 let sequence = 0 + let hasMore = false const locators: string[] = [] /** @@ -83,24 +138,72 @@ export async function runDrain( * background-job logs while `lastRunAt` quietly advances. */ let session: ReturnType | null = null + let redactError = createCredentialErrorRedactor() + + async function checkpoint(status: 'running' | 'success'): Promise { + const now = new Date() + await db.transaction(async (tx) => { + const [currentDrain] = await tx + .select({ cursor: dataDrains.cursor }) + .from(dataDrains) + .where(eq(dataDrains.id, drainId)) + .limit(1) + .for('update') + const [currentRun] = await tx + .select({ status: dataDrainRuns.status }) + .from(dataDrainRuns) + .where(eq(dataDrainRuns.id, runId)) + .limit(1) + .for('update') + if ( + !currentDrain || + currentRun?.status !== 'running' || + currentDrain.cursor !== checkpointCursor + ) { + throw new Error('Data drain run lease lost') + } + await tx + .update(dataDrains) + .set({ + cursor, + ...(status === 'success' ? { lastSuccessAt: now } : {}), + updatedAt: now, + }) + .where(eq(dataDrains.id, drainId)) + await tx + .update(dataDrainRuns) + .set({ + status, + ...(status === 'success' ? { finishedAt: now } : {}), + rowsExported, + bytesWritten, + cursorAfter: cursor, + locators, + error: null, + }) + .where(eq(dataDrainRuns.id, runId)) + }) + checkpointCursor = cursor + } try { const config = destination.configSchema.parse(drain.destinationConfig) - const credentials = destination.credentialsSchema.parse( - await decryptCredentials(drain.destinationCredentials) - ) + const savedCredentials = await decryptCredentials(drain.destinationCredentials) + redactError = createCredentialErrorRedactor(savedCredentials) + const credentials = destination.credentialsSchema.parse(savedCredentials) const activeSession = destination.openSession({ config, credentials }) session = activeSession - for await (const chunk of source.pages({ - organizationId: drain.organizationId, - cursor, - chunkSize: CHUNK_SIZE, - signal, - })) { - const ndjson = `${chunk.map((row) => JSON.stringify(source.serialize(row))).join('\n')}\n` - const body = Buffer.from(ndjson, 'utf8') + let lines: string[] = [] + let chunkBytes = 0 + let chunkCursor: Cursor = cursor + const softDeadline = startedAt.getTime() + DATA_DRAIN_LIMITS.softDurationMs + async function deliverChunk(): Promise { + if (lines.length === 0) return + signal.throwIfAborted() + const rowCount = lines.length + const body = Buffer.from(`${lines.join('\n')}\n`, 'utf8') const result = await runWithOutboundOrganization(drain.organizationId, () => activeSession.deliver({ body, @@ -110,48 +213,79 @@ export async function runDrain( runId, source: drain.source, sequence, - rowCount: chunk.length, + rowCount, runStartedAt: startedAt, }, signal, }) ) - locators.push(result.locator) - rowsExported += chunk.length + rowsExported += rowCount bytesWritten += body.byteLength - cursor = source.cursorAfter(chunk[chunk.length - 1]) + cursor = chunkCursor sequence++ + await checkpoint('running') + lines = [] + chunkBytes = 0 + } + + function runWindowFull(): boolean { + return ( + sequence >= DATA_DRAIN_LIMITS.maxChunksPerRun || + rowsExported >= DATA_DRAIN_LIMITS.maxRowsPerRun || + bytesWritten >= DATA_DRAIN_LIMITS.maxBytesPerRun || + Date.now() >= softDeadline + ) + } + + pages: for await (const chunk of source.pages({ + organizationId: drain.organizationId, + cursor, + chunkSize: DATA_DRAIN_LIMITS.pageRows, + signal, + })) { + for (const row of chunk) { + if (runWindowFull() || rowsExported + lines.length >= DATA_DRAIN_LIMITS.maxRowsPerRun) { + await deliverChunk() + hasMore = true + break pages + } + const line = JSON.stringify(source.serialize(row)) + const rowBytes = Buffer.byteLength(line, 'utf8') + if (rowBytes > DATA_DRAIN_LIMITS.maxRowBytes) { + throw new Error( + `Data drain record is ${rowBytes} bytes, exceeds the ${DATA_DRAIN_LIMITS.maxRowBytes}-byte limit` + ) + } + const lineBytes = rowBytes + 1 + if (chunkBytes + lineBytes > DATA_DRAIN_LIMITS.maxChunkBytes) { + await deliverChunk() + if (runWindowFull()) { + hasMore = true + break pages + } + } + if (bytesWritten + chunkBytes + lineBytes > DATA_DRAIN_LIMITS.maxBytesPerRun) { + await deliverChunk() + hasMore = true + break pages + } + lines.push(line) + chunkBytes += lineBytes + chunkCursor = source.cursorAfter(row) + } + await deliverChunk() + if (runWindowFull()) { + hasMore = true + break + } } if (signal.aborted) { throw new Error('Data drain run cancelled') } - const finishedAt = new Date() - await db.transaction(async (tx) => { - await tx - .update(dataDrains) - .set({ - cursor, - lastRunAt: finishedAt, - lastSuccessAt: finishedAt, - updatedAt: finishedAt, - }) - .where(eq(dataDrains.id, drainId)) - await tx - .update(dataDrainRuns) - .set({ - status: 'success', - finishedAt, - rowsExported, - bytesWritten, - cursorAfter: cursor, - locators, - error: null, - }) - .where(eq(dataDrainRuns.id, runId)) - }) + await checkpoint('success') logger.info('Data drain run succeeded', { drainId, @@ -161,6 +295,7 @@ export async function runDrain( rowsExported, bytesWritten, chunks: sequence, + hasMore, }) return { @@ -172,16 +307,27 @@ export async function runDrain( cursorBefore, cursorAfter: cursor, locators, + hasMore, } } catch (error) { const finishedAt = new Date() - const message = toError(error).message + const message = redactError(error) try { await db.transaction(async (tx) => { await tx - .update(dataDrains) - .set({ lastRunAt: finishedAt, updatedAt: finishedAt }) + .select({ id: dataDrains.id }) + .from(dataDrains) .where(eq(dataDrains.id, drainId)) + .limit(1) + .for('update') + const [currentRun] = await tx + .select({ status: dataDrainRuns.status, cursorAfter: dataDrainRuns.cursorAfter }) + .from(dataDrainRuns) + .where(eq(dataDrainRuns.id, runId)) + .limit(1) + .for('update') + if (currentRun?.status !== 'running') return + await tx.update(dataDrains).set({ updatedAt: finishedAt }).where(eq(dataDrains.id, drainId)) await tx .update(dataDrainRuns) .set({ @@ -189,9 +335,9 @@ export async function runDrain( finishedAt, rowsExported, bytesWritten, - cursorAfter: cursorBefore, + cursorAfter: currentRun.cursorAfter, locators, - error: message.slice(0, 4000), + error: message, }) .where(eq(dataDrainRuns.id, runId)) }) @@ -202,7 +348,7 @@ export async function runDrain( drainId, runId, deliveryError: message, - statusError: toError(statusError).message, + statusError: redactError(statusError), }) } @@ -214,7 +360,7 @@ export async function runDrain( error: message, }) - throw error + throw new Error(message) } finally { if (session) { try { @@ -223,7 +369,7 @@ export async function runDrain( logger.warn('Data drain session close failed', { drainId, runId, - error: toError(closeError).message, + error: redactError(closeError), }) } } diff --git a/apps/sim/lib/data-drains/sources/audit-logs.ts b/apps/sim/lib/data-drains/sources/audit-logs.ts index b94ce50ae8b..69414a243a0 100644 --- a/apps/sim/lib/data-drains/sources/audit-logs.ts +++ b/apps/sim/lib/data-drains/sources/audit-logs.ts @@ -1,6 +1,7 @@ -import { dbReplica } from '@sim/db' import { auditLog } from '@sim/db/schema' -import { and, inArray, isNull, or, sql } from 'drizzle-orm' +import { and, inArray } from 'drizzle-orm' +import { buildOrgScopeCondition } from '@/lib/audit-logs/query' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' import { decodeTimeCursor, encodeTimeCursor, @@ -8,46 +9,47 @@ import { timeCursorPredicate, timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' -import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' +import { readBoundedSourcePage } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' type AuditLogRow = typeof auditLog.$inferSelect /** * Drains audit events scoped to the organization: rows from any of the org's - * workspaces, plus org-level rows (`workspace_id IS NULL`) where - * `metadata->>'organizationId'` matches. Audit-log writers consistently set - * `metadata.organizationId` for org-scoped actions even though the table has - * no dedicated FK column. + * workspaces, plus organization-level rows using the same tenant predicate as + * the audit viewer. Departed actors remain part of the exported security trail. */ async function* pages(input: SourcePageInput): AsyncIterable { - const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId) - - const orgScopedClause = and( - isNull(auditLog.workspaceId), - sql`${auditLog.metadata}->>'organizationId' = ${input.organizationId}` - ) - const scopeClause = - workspaceIds.length === 0 - ? orgScopedClause - : or(inArray(auditLog.workspaceId, workspaceIds), orgScopedClause) + const scopeClause = buildOrgScopeCondition({ + organizationId: input.organizationId, + orgMemberIds: [], + includeDeparted: true, + }) let cursor = decodeTimeCursor(input.cursor) while (!input.signal.aborted) { const cursorClause = timeCursorPredicate(auditLog.createdAt, auditLog.id, cursor) - const rows = await dbReplica - .select() - .from(auditLog) - .where(and(scopeClause, timeCursorStabilityBound(auditLog.createdAt), cursorClause)) - .orderBy(...timeCursorOrderBy(auditLog.createdAt, auditLog.id)) - .limit(input.chunkSize) + const orderBy = timeCursorOrderBy(auditLog.createdAt, auditLog.id) + const rows = await readBoundedSourcePage({ + table: auditLog, + idColumn: auditLog.id, + condition: and(scopeClause, timeCursorStabilityBound(auditLog.createdAt), cursorClause), + orderBy, + chunkSize: input.chunkSize, + read: (tx, ids) => + tx + .select() + .from(auditLog) + .where(inArray(auditLog.id, ids)) + .orderBy(...orderBy), + }) if (rows.length === 0) return yield rows const last = rows[rows.length - 1] cursor = { ts: last.createdAt.toISOString(), id: last.id } - if (rows.length < input.chunkSize) return + if (rows.length < Math.min(input.chunkSize, DATA_DRAIN_LIMITS.pageRows)) return } } @@ -70,6 +72,7 @@ export const auditLogsSource: DrainSource = { metadata: row.metadata, ipAddress: row.ipAddress, userAgent: row.userAgent, + surface: row.surface, createdAt: row.createdAt.toISOString(), } }, diff --git a/apps/sim/lib/data-drains/sources/copilot-chats.ts b/apps/sim/lib/data-drains/sources/copilot-chats.ts index 9e5af27ae3b..da2ccc7409f 100644 --- a/apps/sim/lib/data-drains/sources/copilot-chats.ts +++ b/apps/sim/lib/data-drains/sources/copilot-chats.ts @@ -1,6 +1,7 @@ -import { dbReplica } from '@sim/db' import { copilotChats, copilotMessages } from '@sim/db/schema' -import { and, asc, inArray, isNull, sql } from 'drizzle-orm' +import { and, asc, eq, inArray, isNull, or, sql } from 'drizzle-orm' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' import { decodeTimeCursor, encodeTimeCursor, @@ -8,7 +9,7 @@ import { timeCursorPredicate, timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' -import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' +import { readBoundedSourcePage, workspaceInOrganization } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' /** @@ -59,55 +60,84 @@ const chatColumns = { * design and `data-drains` is not a CDC pipeline. */ async function* pages(input: SourcePageInput): AsyncIterable { - const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId) - if (workspaceIds.length === 0) return - let cursor = decodeTimeCursor(input.cursor) while (!input.signal.aborted) { const cursorClause = timeCursorPredicate(copilotChats.createdAt, copilotChats.id, cursor) - - const metaRows = await dbReplica - .select(chatColumns) - .from(copilotChats) - .where( - and( - inArray(copilotChats.workspaceId, workspaceIds), - timeCursorStabilityBound(copilotChats.createdAt), - cursorClause + const orderBy = timeCursorOrderBy(copilotChats.createdAt, copilotChats.id) + const rows = await readBoundedSourcePage({ + table: copilotChats, + idColumn: copilotChats.id, + condition: and( + or( + workspaceInOrganization(copilotChats.workspaceId, input.organizationId), + and( + isNull(copilotChats.workspaceId), + eq(copilotChats.organizationId, input.organizationId) + ) + ), + timeCursorStabilityBound(copilotChats.createdAt), + cursorClause + ), + orderBy, + chunkSize: input.chunkSize, + measuredValue: sql`json_build_object(${sql.join( + Object.entries(chatColumns).flatMap(([name, column]) => [ + sql`${name}::text`, + sql`${column}`, + ]), + sql`, ` + )})`, + read: async (tx, ids) => { + const metaRows = await tx + .select(chatColumns) + .from(copilotChats) + .where(inArray(copilotChats.id, ids)) + .orderBy(...orderBy) + const messageCondition = and( + inArray(copilotMessages.chatId, ids), + isNull(copilotMessages.deletedAt) ) - ) - .orderBy(...timeCursorOrderBy(copilotChats.createdAt, copilotChats.id)) - .limit(input.chunkSize) - - if (metaRows.length === 0) return - - const chatIds = metaRows.map((r) => r.id) - const messageRows = await dbReplica - .select({ chatId: copilotMessages.chatId, content: copilotMessages.content }) - .from(copilotMessages) - .where(and(inArray(copilotMessages.chatId, chatIds), isNull(copilotMessages.deletedAt))) - .orderBy( - asc(copilotMessages.chatId), - sql`${copilotMessages.seq} asc nulls last`, - asc(copilotMessages.createdAt), - asc(copilotMessages.id) - ) - const messagesByChat = new Map() - for (const m of messageRows) { - const existing = messagesByChat.get(m.chatId) - if (existing) existing.push(m.content) - else messagesByChat.set(m.chatId, [m.content]) - } - - const rows: CopilotChatRow[] = metaRows.map((r) => ({ - ...r, - messages: messagesByChat.get(r.id) ?? [], - })) - + const transcriptSizes = await tx + .select({ + chatId: copilotMessages.chatId, + bytes: sql`sum(octet_length(row_to_json(${copilotMessages})::text))`.mapWith( + Number + ), + }) + .from(copilotMessages) + .where(messageCondition) + .groupBy(copilotMessages.chatId) + for (const transcript of transcriptSizes) { + assertKnownSizeWithinLimit( + transcript.bytes, + DATA_DRAIN_LIMITS.maxRowBytes, + 'Drain transcript record' + ) + } + const messageRows = await tx + .select({ chatId: copilotMessages.chatId, content: copilotMessages.content }) + .from(copilotMessages) + .where(messageCondition) + .orderBy( + asc(copilotMessages.chatId), + sql`${copilotMessages.seq} asc nulls last`, + asc(copilotMessages.createdAt), + asc(copilotMessages.id) + ) + const messagesByChat = new Map() + for (const message of messageRows) { + const existing = messagesByChat.get(message.chatId) + if (existing) existing.push(message.content) + else messagesByChat.set(message.chatId, [message.content]) + } + return metaRows.map((row) => ({ ...row, messages: messagesByChat.get(row.id) ?? [] })) + }, + }) + if (rows.length === 0) return yield rows - const last = metaRows[metaRows.length - 1] + const last = rows[rows.length - 1] cursor = { ts: last.createdAt.toISOString(), id: last.id } - if (metaRows.length < input.chunkSize) return + if (rows.length < Math.min(input.chunkSize, DATA_DRAIN_LIMITS.pageRows)) return } } diff --git a/apps/sim/lib/data-drains/sources/copilot-runs.ts b/apps/sim/lib/data-drains/sources/copilot-runs.ts index b6270c09456..60da8363cd0 100644 --- a/apps/sim/lib/data-drains/sources/copilot-runs.ts +++ b/apps/sim/lib/data-drains/sources/copilot-runs.ts @@ -1,6 +1,7 @@ -import { dbReplica } from '@sim/db' import { copilotRuns } from '@sim/db/schema' -import { and, inArray, isNotNull } from 'drizzle-orm' +import { isRecordLike } from '@sim/utils/object' +import { and, eq, inArray, isNotNull, isNull, or } from 'drizzle-orm' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' import { decodeTimeCursor, encodeTimeCursor, @@ -8,7 +9,7 @@ import { timeCursorPredicate, timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' -import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' +import { readBoundedSourcePage, workspaceInOrganization } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' type CopilotRunRow = typeof copilotRuns.$inferSelect @@ -18,32 +19,37 @@ type CopilotRunRow = typeof copilotRuns.$inferSelect * `error`, `completedAt`) are not exported until they reach a terminal state. */ async function* pages(input: SourcePageInput): AsyncIterable { - const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId) - if (workspaceIds.length === 0) return - let cursor = decodeTimeCursor(input.cursor) while (!input.signal.aborted) { const cursorClause = timeCursorPredicate(copilotRuns.completedAt, copilotRuns.id, cursor) - - const rows = await dbReplica - .select() - .from(copilotRuns) - .where( - and( - inArray(copilotRuns.workspaceId, workspaceIds), - isNotNull(copilotRuns.completedAt), - timeCursorStabilityBound(copilotRuns.completedAt), - cursorClause - ) - ) - .orderBy(...timeCursorOrderBy(copilotRuns.completedAt, copilotRuns.id)) - .limit(input.chunkSize) + const orderBy = timeCursorOrderBy(copilotRuns.completedAt, copilotRuns.id) + const rows = await readBoundedSourcePage({ + table: copilotRuns, + idColumn: copilotRuns.id, + condition: and( + or( + workspaceInOrganization(copilotRuns.workspaceId, input.organizationId), + and(isNull(copilotRuns.workspaceId), eq(copilotRuns.organizationId, input.organizationId)) + ), + isNotNull(copilotRuns.completedAt), + timeCursorStabilityBound(copilotRuns.completedAt), + cursorClause + ), + orderBy, + chunkSize: input.chunkSize, + read: (tx, ids) => + tx + .select() + .from(copilotRuns) + .where(inArray(copilotRuns.id, ids)) + .orderBy(...orderBy), + }) if (rows.length === 0) return yield rows const last = rows[rows.length - 1] cursor = { ts: last.completedAt!.toISOString(), id: last.id } - if (rows.length < input.chunkSize) return + if (rows.length < Math.min(input.chunkSize, DATA_DRAIN_LIMITS.pageRows)) return } } @@ -65,7 +71,16 @@ export const copilotRunsSource: DrainSource = { model: row.model, provider: row.provider, status: row.status, - requestContext: row.requestContext, + requestContext: isRecordLike(row.requestContext) + ? { + ...(typeof row.requestContext.requestId === 'string' + ? { requestId: row.requestContext.requestId.slice(0, 128) } + : {}), + ...(typeof row.requestContext.source === 'string' + ? { source: row.requestContext.source.slice(0, 128) } + : {}), + } + : null, startedAt: row.startedAt.toISOString(), completedAt: row.completedAt ? row.completedAt.toISOString() : null, createdAt: row.createdAt.toISOString(), diff --git a/apps/sim/lib/data-drains/sources/coverage.integration.ts b/apps/sim/lib/data-drains/sources/coverage.integration.ts new file mode 100644 index 00000000000..9ff1afde36e --- /dev/null +++ b/apps/sim/lib/data-drains/sources/coverage.integration.ts @@ -0,0 +1,196 @@ +import { generateId } from '@sim/utils/id' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +async function loadRuntime() { + const [ + { db }, + schema, + { eq, inArray }, + { auditLogsSource }, + { copilotChatsSource }, + { copilotRunsSource }, + ] = await Promise.all([ + import('@sim/db'), + import('@sim/db/schema'), + import('drizzle-orm'), + import('@/lib/data-drains/sources/audit-logs'), + import('@/lib/data-drains/sources/copilot-chats'), + import('@/lib/data-drains/sources/copilot-runs'), + ]) + return { db, schema, eq, inArray, auditLogsSource, copilotChatsSource, copilotRunsSource } +} + +describe('Drain source coverage and payload safety in PostgreSQL', () => { + let runtime: Awaited> + const userId = generateId() + const organizationId = generateId() + const otherOrganizationId = generateId() + const chatId = generateId() + const otherChatId = generateId() + const oversizedEventId = generateId() + const runId = generateId() + const old = new Date('2026-01-01T00:00:00.000Z') + + beforeAll(async () => { + runtime = await loadRuntime() + const { db, schema } = runtime + await db.insert(schema.user).values({ + id: userId, + name: 'Drain fixture', + email: `${userId}@drain-source.test`, + emailVerified: true, + createdAt: old, + updatedAt: old, + }) + await db.insert(schema.organization).values( + [organizationId, otherOrganizationId].map((id) => ({ + id, + name: 'Drain fixture', + slug: id, + createdAt: old, + })) + ) + await db.insert(schema.copilotChats).values([ + { id: chatId, userId, organizationId, createdAt: old, updatedAt: old }, + { + id: otherChatId, + userId, + organizationId: otherOrganizationId, + createdAt: old, + updatedAt: old, + }, + ]) + await db.insert(schema.copilotMessages).values({ + id: generateId(), + messageId: generateId(), + chatId, + role: 'user', + content: { text: 'Organization chat' }, + createdAt: old, + }) + await db.insert(schema.copilotRuns).values({ + id: runId, + chatId, + userId, + organizationId, + executionId: generateId(), + streamId: generateId(), + status: 'complete', + startedAt: old, + completedAt: old, + createdAt: old, + updatedAt: old, + requestContext: { + requestId: 'public-correlation-id', + controllerToken: 'private-controller-token', + recovery: { billingAdmission: { serializedAttribution: 'private-billing-receipt' } }, + }, + }) + }, 30_000) + + afterAll(async () => { + if (!runtime) return + const { db, schema, eq, inArray } = runtime + await db.delete(schema.auditLog).where(eq(schema.auditLog.id, oversizedEventId)) + await db.delete(schema.user).where(eq(schema.user.id, userId)) + await db + .delete(schema.organization) + .where(inArray(schema.organization.id, [organizationId, otherOrganizationId])) + }) + + function input() { + return { organizationId, cursor: null, chunkSize: 100, signal: new AbortController().signal } + } + + it('includes organization-owned chats and runs without including another tenant', async () => { + const chats: string[] = [] + for await (const rows of runtime.copilotChatsSource.pages(input())) + chats.push(...rows.map((row) => row.id)) + expect(chats).toEqual([chatId]) + const runs: string[] = [] + for await (const rows of runtime.copilotRunsSource.pages(input())) + runs.push(...rows.map((row) => row.id)) + expect(runs).toEqual([runId]) + }) + + it('exports correlation metadata without internal controller tokens or billing receipts', async () => { + const [row] = await runtime.db + .select() + .from(runtime.schema.copilotRuns) + .where(runtime.eq(runtime.schema.copilotRuns.id, runId)) + const exported = runtime.copilotRunsSource.serialize(row) + expect(exported.requestContext).toEqual({ requestId: 'public-correlation-id' }) + expect(JSON.stringify(exported)).not.toContain('private-controller-token') + expect(JSON.stringify(exported)).not.toContain('private-billing-receipt') + }) + + it('refuses an oversized transcript before loading its messages', async () => { + const messageId = generateId() + await runtime.db.insert(runtime.schema.copilotMessages).values({ + id: messageId, + messageId: generateId(), + chatId, + role: 'user', + content: { text: 'x'.repeat(1024 * 1024 + 1) }, + createdAt: old, + }) + try { + await expect(async () => { + for await (const _rows of runtime.copilotChatsSource.pages(input())) { + } + }).rejects.toThrow(/record.*(bytes|size|limit)/i) + } finally { + await runtime.db + .delete(runtime.schema.copilotMessages) + .where(runtime.eq(runtime.schema.copilotMessages.id, messageId)) + } + }) + + it('keeps paging when a caller requests more rows than the source page budget', async () => { + const scope = generateId() + const eventIds = Array.from({ length: 101 }, () => generateId()) + await runtime.db.insert(runtime.schema.auditLog).values( + eventIds.map((id) => ({ + id, + actorId: null, + action: 'organization.updated', + resourceType: 'organization', + resourceId: scope, + metadata: { organizationId: scope }, + createdAt: old, + })) + ) + try { + const seen: string[] = [] + for await (const rows of runtime.auditLogsSource.pages({ + ...input(), + organizationId: scope, + chunkSize: 1000, + })) { + seen.push(...rows.map((row) => row.id)) + } + expect(new Set(seen)).toEqual(new Set(eventIds)) + expect(seen).toHaveLength(eventIds.length) + } finally { + await runtime.db + .delete(runtime.schema.auditLog) + .where(runtime.inArray(runtime.schema.auditLog.id, eventIds)) + } + }) + + it('refuses an oversized database record at the source boundary', async () => { + await runtime.db.insert(runtime.schema.auditLog).values({ + id: oversizedEventId, + actorId: null, + action: 'organization.updated', + resourceType: 'organization', + resourceId: organizationId, + createdAt: old, + metadata: { organizationId, value: 'x'.repeat(1024 * 1024 + 1) }, + }) + await expect(async () => { + for await (const _rows of runtime.auditLogsSource.pages(input())) { + } + }).rejects.toThrow(/record.*(bytes|size|limit)/i) + }) +}) diff --git a/apps/sim/lib/data-drains/sources/helpers.ts b/apps/sim/lib/data-drains/sources/helpers.ts index 33ff90ab42c..065faccc0dd 100644 --- a/apps/sim/lib/data-drains/sources/helpers.ts +++ b/apps/sim/lib/data-drains/sources/helpers.ts @@ -1,15 +1,53 @@ import { db } from '@sim/db' import { workspace } from '@sim/db/schema' -import { eq } from 'drizzle-orm' +import { type SQL, sql } from 'drizzle-orm' +import type { PgColumn, PgTable } from 'drizzle-orm/pg-core' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' -/** - * Returns the IDs of all workspaces belonging to the organization. Used by - * sources whose underlying tables are workspace-scoped rather than org-scoped. - */ -export async function getOrganizationWorkspaceIds(organizationId: string): Promise { - const rows = await db - .select({ id: workspace.id }) - .from(workspace) - .where(eq(workspace.organizationId, organizationId)) - return rows.map((row) => row.id) +type SourceTransaction = Parameters[0]>[0] + +/** Tests workspace ownership in SQL without loading the organization's workspace IDs. */ +export function workspaceInOrganization(column: PgColumn, organizationId: string): SQL { + return sql`exists ( + select 1 from ${workspace} + where ${workspace.id} = ${column} and ${workspace.organizationId} = ${organizationId} + )` +} + +/** Measures database payloads before loading them, under the same immutable read snapshot. */ +export async function readBoundedSourcePage(input: { + table: PgTable + idColumn: PgColumn + condition: SQL | undefined + orderBy: SQL[] + chunkSize: number + measuredValue?: SQL + read(tx: SourceTransaction, ids: string[]): Promise +}): Promise { + return db.transaction( + async (tx) => { + const candidates = await tx + .select({ + id: sql`${input.idColumn}`, + bytes: + sql`octet_length((${input.measuredValue ?? sql`row_to_json(${input.table})`})::text)`.mapWith( + Number + ), + }) + .from(input.table) + .where(input.condition) + .orderBy(...input.orderBy) + .limit(Math.min(input.chunkSize, DATA_DRAIN_LIMITS.pageRows)) + for (const candidate of candidates) { + assertKnownSizeWithinLimit(candidate.bytes, DATA_DRAIN_LIMITS.maxRowBytes, 'Drain record') + } + if (candidates.length === 0) return [] + return input.read( + tx, + candidates.map((row) => row.id) + ) + }, + { isolationLevel: 'repeatable read', accessMode: 'read only' } + ) } diff --git a/apps/sim/lib/data-drains/sources/job-logs.ts b/apps/sim/lib/data-drains/sources/job-logs.ts index d842fefbfa8..f2071d5bbe6 100644 --- a/apps/sim/lib/data-drains/sources/job-logs.ts +++ b/apps/sim/lib/data-drains/sources/job-logs.ts @@ -1,7 +1,7 @@ -import { dbReplica } from '@sim/db' import { jobExecutionLogs } from '@sim/db/schema' import { and, inArray, isNotNull } from 'drizzle-orm' -import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' import { decodeTimeCursor, encodeTimeCursor, @@ -9,7 +9,7 @@ import { timeCursorPredicate, timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' -import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' +import { readBoundedSourcePage, workspaceInOrganization } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' @@ -20,33 +20,37 @@ type JobLogRow = typeof jobExecutionLogs.$inferSelect * `totalDurationMs`, `executionData`) are not exported until finalized. */ async function* pages(input: SourcePageInput): AsyncIterable { - const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId) - if (workspaceIds.length === 0) return - let cursor = decodeTimeCursor(input.cursor) while (!input.signal.aborted) { const cursorClause = timeCursorPredicate(jobExecutionLogs.endedAt, jobExecutionLogs.id, cursor) - const rows = await dbReplica - .select() - .from(jobExecutionLogs) - .where( - and( - inArray(jobExecutionLogs.workspaceId, workspaceIds), - isNotNull(jobExecutionLogs.endedAt), - timeCursorStabilityBound(jobExecutionLogs.endedAt), - cursorClause - ) - ) - .orderBy(...timeCursorOrderBy(jobExecutionLogs.endedAt, jobExecutionLogs.id)) - .limit(input.chunkSize) + const orderBy = timeCursorOrderBy(jobExecutionLogs.endedAt, jobExecutionLogs.id) + const rows = await readBoundedSourcePage({ + table: jobExecutionLogs, + idColumn: jobExecutionLogs.id, + condition: and( + workspaceInOrganization(jobExecutionLogs.workspaceId, input.organizationId), + isNotNull(jobExecutionLogs.endedAt), + timeCursorStabilityBound(jobExecutionLogs.endedAt), + cursorClause + ), + orderBy, + chunkSize: input.chunkSize, + read: (tx, ids) => + tx + .select() + .from(jobExecutionLogs) + .where(inArray(jobExecutionLogs.id, ids)) + .orderBy(...orderBy), + }) if (rows.length === 0) return - const displayExecutionData = await mapWithConcurrency(rows, MATERIALIZE_CONCURRENCY, (row) => + const displayExecutionData = await mapWithConcurrency(rows, 2, (row) => materializeExecutionDataForDisplay(row.executionData as Record | null, { workspaceId: row.workspaceId, workflowId: null, executionId: row.executionId, + maxBytes: DATA_DRAIN_LIMITS.maxRowBytes, }) ) for (let index = 0; index < rows.length; index += 1) { @@ -55,7 +59,7 @@ async function* pages(input: SourcePageInput): AsyncIterable { yield rows const last = rows[rows.length - 1] cursor = { ts: last.endedAt!.toISOString(), id: last.id } - if (rows.length < input.chunkSize) return + if (rows.length < Math.min(input.chunkSize, DATA_DRAIN_LIMITS.pageRows)) return } } diff --git a/apps/sim/lib/data-drains/sources/workflow-logs.ts b/apps/sim/lib/data-drains/sources/workflow-logs.ts index a0f7b9e14ae..6ccab85cee7 100644 --- a/apps/sim/lib/data-drains/sources/workflow-logs.ts +++ b/apps/sim/lib/data-drains/sources/workflow-logs.ts @@ -1,7 +1,7 @@ -import { dbReplica } from '@sim/db' import { workflowExecutionLogs } from '@sim/db/schema' import { and, inArray, isNotNull } from 'drizzle-orm' -import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' import { decodeTimeCursor, encodeTimeCursor, @@ -9,7 +9,7 @@ import { timeCursorPredicate, timeCursorStabilityBound, } from '@/lib/data-drains/sources/cursor' -import { getOrganizationWorkspaceIds } from '@/lib/data-drains/sources/helpers' +import { readBoundedSourcePage, workspaceInOrganization } from '@/lib/data-drains/sources/helpers' import type { Cursor, DrainSource, SourcePageInput } from '@/lib/data-drains/types' import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' @@ -23,9 +23,6 @@ type WorkflowLogRow = Omit * once visible to the drain. */ async function* pages(input: SourcePageInput): AsyncIterable { - const workspaceIds = await getOrganizationWorkspaceIds(input.organizationId) - if (workspaceIds.length === 0) return - let cursor = decodeTimeCursor(input.cursor) while (!input.signal.aborted) { const cursorClause = timeCursorPredicate( @@ -34,19 +31,25 @@ async function* pages(input: SourcePageInput): AsyncIterable { cursor ) - const rows = await dbReplica - .select() - .from(workflowExecutionLogs) - .where( - and( - inArray(workflowExecutionLogs.workspaceId, workspaceIds), - isNotNull(workflowExecutionLogs.endedAt), - timeCursorStabilityBound(workflowExecutionLogs.endedAt), - cursorClause - ) - ) - .orderBy(...timeCursorOrderBy(workflowExecutionLogs.endedAt, workflowExecutionLogs.id)) - .limit(input.chunkSize) + const orderBy = timeCursorOrderBy(workflowExecutionLogs.endedAt, workflowExecutionLogs.id) + const rows = await readBoundedSourcePage({ + table: workflowExecutionLogs, + idColumn: workflowExecutionLogs.id, + condition: and( + workspaceInOrganization(workflowExecutionLogs.workspaceId, input.organizationId), + isNotNull(workflowExecutionLogs.endedAt), + timeCursorStabilityBound(workflowExecutionLogs.endedAt), + cursorClause + ), + orderBy, + chunkSize: input.chunkSize, + read: (tx, ids) => + tx + .select() + .from(workflowExecutionLogs) + .where(inArray(workflowExecutionLogs.id, ids)) + .orderBy(...orderBy), + }) if (rows.length === 0) return @@ -54,11 +57,12 @@ async function* pages(input: SourcePageInput): AsyncIterable { // concurrency) so the drain exports full execution data, not the slim row. // Use the order-preserving returned array (the util's documented contract) // and write back, rather than mutating rows inside the mapper. - const materialized = await mapWithConcurrency(rows, MATERIALIZE_CONCURRENCY, (row) => + const materialized = await mapWithConcurrency(rows, 2, (row) => materializeExecutionDataForDisplay(row.executionData as Record | null, { workspaceId: row.workspaceId, workflowId: row.workflowId, executionId: row.executionId, + maxBytes: DATA_DRAIN_LIMITS.maxRowBytes, }) ) for (let i = 0; i < rows.length; i++) { @@ -68,7 +72,7 @@ async function* pages(input: SourcePageInput): AsyncIterable { yield rows const last = rows[rows.length - 1] cursor = { ts: last.endedAt!.toISOString(), id: last.id } - if (rows.length < input.chunkSize) return + if (rows.length < Math.min(input.chunkSize, DATA_DRAIN_LIMITS.pageRows)) return } } diff --git a/apps/sim/lib/logs/execution/trace-store.test.ts b/apps/sim/lib/logs/execution/trace-store.test.ts index 1a939c5f9dd..3e0a53c2d48 100644 --- a/apps/sim/lib/logs/execution/trace-store.test.ts +++ b/apps/sim/lib/logs/execution/trace-store.test.ts @@ -161,6 +161,42 @@ describe('execution data storage', () => { ) }) +describe('bounded drain archive reads', () => { + it('rejects a drain archive whose authorization scope cannot be resolved', async () => { + const data = { + [TRACE_STORE_REF_KEY]: { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'object', + size: 128, + key: 'execution/execution-1/large-value-lv_bbbbbbbbbbbb.json', + executionId: 'execution-1', + }, + } + await expect( + materializeExecutionData(data, { ...CONTEXT, workflowId: null, maxBytes: 1024 }) + ).rejects.toThrow(/scope|workflow/i) + }) + + it('rejects an archive above the caller budget instead of returning an incomplete success', async () => { + const data = { + [TRACE_STORE_REF_KEY]: { + __simLargeValueRef: true, + version: 1, + id: 'lv_bbbbbbbbbbbb', + kind: 'object', + size: 2048, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_bbbbbbbbbbbb.json', + executionId: 'execution-1', + }, + } + await expect(materializeExecutionData(data, { ...CONTEXT, maxBytes: 1024 })).rejects.toThrow( + /record.*(bytes|size|limit)/i + ) + }) +}) + describe('projectExecutionDataForDisplay', () => { it('projects authoritative state-only block outputs without mutating execution state', async () => { const executionData = { diff --git a/apps/sim/lib/logs/execution/trace-store.ts b/apps/sim/lib/logs/execution/trace-store.ts index 03aabd7258a..78ae6fae575 100644 --- a/apps/sim/lib/logs/execution/trace-store.ts +++ b/apps/sim/lib/logs/execution/trace-store.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { describeError, toError } from '@sim/utils/errors' import { isRecordLike, omit } from '@sim/utils/object' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' import { MAX_TRACE_ARCHIVE_BYTES } from '@/lib/execution/payloads/limits' import { @@ -76,6 +77,8 @@ export interface TraceStoreReadContext { workflowId: string | null executionId: string userId?: string + /** A strict caller budget: oversized or unavailable archives fail instead of degrading to metadata. */ + maxBytes?: number } export interface DisplayExecutionDataWithBlockOutputs { @@ -295,7 +298,8 @@ export async function externalizeExecutionData( * `__simLargeValueRef` stubs remain as previews, matching prior behavior. * * Returns metadata-only (the slim row minus the pointer) if the object is - * missing/unreadable (e.g. post-retention) so reads degrade rather than crash. + * missing/unreadable (e.g. post-retention) so ordinary reads degrade. A caller + * supplying `maxBytes` instead requires an available, scoped, bounded archive. */ export async function materializeExecutionData( executionData: Record | null | undefined, @@ -305,29 +309,41 @@ export async function materializeExecutionData( const ref = executionData[TRACE_STORE_REF_KEY] if (!isLargeValueRef(ref)) return executionData + if (context.maxBytes !== undefined) { + assertKnownSizeWithinLimit(ref.size, context.maxBytes, 'Drain archive record') + } const { [TRACE_STORE_REF_KEY]: _pointer, ...markers } = executionData - if (!context.workspaceId) return markers + if (!context.workspaceId) { + if (context.maxBytes !== undefined) + throw new Error('Drain archive workspace scope is unavailable') + return markers + } // workflowId is `set null` on workflow delete, but the ref key embeds the // original workflowId — recover it so deleted-workflow logs stay readable. // Workspace authorization still comes from the (authorized) caller context. const workflowId = context.workflowId ?? workflowIdFromStorageKey(ref.key) - if (!workflowId) return markers + if (!workflowId) { + if (context.maxBytes !== undefined) + throw new Error('Drain archive workflow scope is unavailable') + return markers + } try { const materialized = await materializeLargeValueRef(ref, { workspaceId: context.workspaceId, workflowId, executionId: context.executionId, - maxBytes: Math.min(ref.size, MAX_TRACE_ARCHIVE_BYTES), + maxBytes: Math.min(ref.size, context.maxBytes ?? MAX_TRACE_ARCHIVE_BYTES), // Read-only: the value is already referenced by its own execution; don't // re-register (or fail) on every view/export. trackReference: false, }) if (!materialized || typeof materialized !== 'object') { + if (context.maxBytes !== undefined) throw new Error('Drain execution archive is unavailable') logger.warn('Trace store object unavailable; returning metadata only', { executionId: context.executionId, key: ref.key, @@ -337,6 +353,7 @@ export async function materializeExecutionData( return { ...(materialized as Record), ...markers } } catch (error) { + if (context.maxBytes !== undefined) throw error logger.warn('Failed to materialize execution data; returning metadata only', { executionId: context.executionId, error: toError(error).message, diff --git a/apps/sim/lib/table/application/row-audit.integration.ts b/apps/sim/lib/table/application/row-audit.integration.ts new file mode 100644 index 00000000000..4aebd85ad50 --- /dev/null +++ b/apps/sim/lib/table/application/row-audit.integration.ts @@ -0,0 +1,175 @@ +import { db } from '@sim/db' +import { + auditLog, + permissions, + user, + userTableDefinitions, + userTableRows, + workspace, +} from '@sim/db/schema' +import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' +import { storageServiceMock } from '@sim/testing/mocks/storage-service.mock' +import { tableBillingMock, tableBillingMockFns } from '@sim/testing/mocks/table-billing.mock' +import { tableEventsMock } from '@sim/testing/mocks/table-events.mock' +import { tableTriggerMock } from '@sim/testing/mocks/table-trigger.mock' +import { tableWorkflowColumnsMock } from '@sim/testing/mocks/table-workflow-columns.mock' +import { generateId } from '@sim/utils/id' +import { eq } from 'drizzle-orm' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/table/billing', () => tableBillingMock) +vi.mock('@/lib/table/events', () => tableEventsMock) +vi.mock('@/lib/table/trigger', () => tableTriggerMock) +vi.mock('@/lib/table/workflow-columns', () => tableWorkflowColumnsMock) +vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) + +import { + createTableRows, + deleteTableRow, + replaceTableRows, + updateTableRow, + updateTableRows, + upsertTableRow, +} from '@/lib/table/application/rows' + +describe('table mutation audit history in PostgreSQL', () => { + let userId: string + let workspaceId: string + let tableId: string + let rowId: string + const privateValue = 'table-cell-must-not-enter-audit-metadata' + + beforeEach(async () => { + userId = generateId() + workspaceId = generateId() + tableId = generateId() + rowId = generateId() + const now = new Date() + tableBillingMockFns.mockAssertRowCapacity.mockResolvedValue(10_000) + await db.insert(user).values({ + id: userId, + name: 'Table audit fixture', + email: `${userId}@audit.test`, + emailVerified: true, + createdAt: now, + updatedAt: now, + }) + await db.insert(workspace).values({ + id: workspaceId, + name: 'Table audit fixture', + ownerId: userId, + billedAccountUserId: userId, + }) + await db.insert(permissions).values({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: workspaceId, + permissionType: 'admin', + }) + await db.insert(userTableDefinitions).values({ + id: tableId, + workspaceId, + name: 'Audited table', + createdBy: userId, + schema: { columns: [{ id: 'value', name: 'value', type: 'string', unique: true }] }, + }) + await db.insert(userTableRows).values({ + id: rowId, + tableId, + workspaceId, + data: { value: 'original' }, + orderKey: 'a0', + }) + }) + + afterEach(async () => { + await db.delete(auditLog).where(eq(auditLog.workspaceId, workspaceId)) + await db.delete(workspace).where(eq(workspace.id, workspaceId)) + await db.delete(user).where(eq(user.id, userId)) + }) + + const cases = [ + { + name: 'single insert', + operation: 'tables.rows.create', + countField: 'rowsInserted', + count: 1, + }, + { name: 'batch insert', operation: 'tables.rows.create', countField: 'rowsInserted', count: 2 }, + { name: 'single update', operation: 'tables.rows.update', countField: 'rowsUpdated', count: 1 }, + { + name: 'filter update', + operation: 'tables.rows.update_many', + countField: 'rowsUpdated', + count: 1, + }, + { name: 'single delete', operation: 'tables.rows.delete', countField: 'rowsDeleted', count: 1 }, + { name: 'replace', operation: 'tables.rows.replace', countField: 'rowsInserted', count: 1 }, + { + name: 'upsert insert', + operation: 'tables.rows.upsert', + countField: 'rowsInserted', + count: 1, + }, + { name: 'upsert update', operation: 'tables.rows.upsert', countField: 'rowsUpdated', count: 1 }, + ] as const + + it.each(cases)('persists a scoped, content-free record for $name', async (testCase) => { + const principal = createSessionPrincipal({ userId }) + const common = { tableId, assertedWorkspaceId: workspaceId } + const write = { ...common, strictWrite: true, dataKeying: 'ids' as const } + if (testCase.name === 'single insert' || testCase.name === 'batch insert') { + await createTableRows.execute({ + principal, + input: + testCase.name === 'single insert' + ? { ...write, kind: 'single', data: { value: privateValue } } + : { ...write, kind: 'batch', rows: [{ value: privateValue }, { value: 'another' }] }, + }) + } else if (testCase.name === 'single update') { + await updateTableRow.execute({ + principal, + input: { ...write, rowId, data: { value: privateValue } }, + }) + } else if (testCase.name === 'filter update') { + await updateTableRows.execute({ + principal, + input: { ...write, filter: { value: 'original' }, data: { value: privateValue } }, + }) + } else if (testCase.name === 'single delete') { + await deleteTableRow.execute({ principal, input: { ...common, rowId } }) + } else if (testCase.name === 'replace') { + await replaceTableRows.execute({ + principal, + input: { ...write, rows: [{ value: privateValue }] }, + }) + } else { + await upsertTableRow.execute({ + principal, + input: { + ...write, + conflictTarget: 'value', + data: { value: testCase.name === 'upsert update' ? 'original' : privateValue }, + }, + }) + } + + await expect + .poll(async () => { + const rows = await db.select().from(auditLog).where(eq(auditLog.workspaceId, workspaceId)) + return rows.length + }) + .toBe(1) + const [entry] = await db.select().from(auditLog).where(eq(auditLog.workspaceId, workspaceId)) + expect(entry).toMatchObject({ + actorId: userId, + workspaceId, + action: 'table.updated', + resourceType: 'table', + resourceId: tableId, + metadata: { operation: testCase.operation, [testCase.countField]: testCase.count }, + }) + expect(JSON.stringify(entry)).not.toContain(privateValue) + }) +}) diff --git a/apps/sim/lib/table/application/rows.ts b/apps/sim/lib/table/application/rows.ts index 222fa9417a7..a113effb4be 100644 --- a/apps/sim/lib/table/application/rows.ts +++ b/apps/sim/lib/table/application/rows.ts @@ -975,6 +975,22 @@ export const createTableRows = defineAuthorizedTableUseCase({ }), } }, + projectAudit: ({ context, result }) => { + const inserted = result.kind === 'single' ? 1 : result.rows.length + if (inserted === 0) return [] + return { + action: AuditAction.TABLE_UPDATED, + resourceType: AuditResourceType.TABLE, + resourceId: context.tableId, + resourceName: context.table.name, + description: `Inserted ${inserted} row(s) into table "${context.table.name}"`, + metadata: { + op: result.kind === 'single' ? 'insert' : 'batch_insert', + rowsInserted: inserted, + ...(result.kind === 'single' ? { rowId: result.row.id } : {}), + }, + } + }, afterSuccess: ({ context, input, result }) => { // Narrowed on the input, not the result: only the single-row variant carries // an actor, and the two discriminants always agree. @@ -1029,6 +1045,21 @@ export const replaceTableRows = defineAuthorizedTableUseCase({ ) return { table: context.table, ...result } }, + projectAudit: ({ context, result }) => { + if (result.deletedCount === 0 && result.insertedCount === 0) return [] + return { + action: AuditAction.TABLE_UPDATED, + resourceType: AuditResourceType.TABLE, + resourceId: context.tableId, + resourceName: context.table.name, + description: `Replaced rows in table "${context.table.name}"`, + metadata: { + op: 'replace_rows', + rowsDeleted: result.deletedCount, + rowsInserted: result.insertedCount, + }, + } + }, afterSuccess: ({ context, result }) => { if (result.deletedCount > 0 || result.insertedCount > 0) { signalTableRowsChanged(context.tableId) @@ -1282,6 +1313,17 @@ export const updateTableRow = defineAuthorizedTableUseCase({ }), } }, + projectAudit: ({ context, result }) => { + if (!result.changed) return [] + return { + action: AuditAction.TABLE_UPDATED, + resourceType: AuditResourceType.TABLE, + resourceId: context.tableId, + resourceName: context.table.name, + description: `Updated a row in table "${context.table.name}"`, + metadata: { op: 'update', rowId: result.row.id, rowsUpdated: 1 }, + } + }, afterSuccess: ({ context, input, result }) => { if (result.changed) signalTableRowsChangedByActor(context.tableId, input.actorClientId) }, @@ -1336,6 +1378,17 @@ export const updateTableRows = defineAuthorizedTableUseCase({ rethrowQueryValidation(error) } }, + projectAudit: ({ context, result }) => { + if (result.affectedCount === 0) return [] + return { + action: AuditAction.TABLE_UPDATED, + resourceType: AuditResourceType.TABLE, + resourceId: context.tableId, + resourceName: context.table.name, + description: `Updated ${result.affectedCount} row(s) in table "${context.table.name}"`, + metadata: { op: 'update_by_filter', rowsUpdated: result.affectedCount }, + } + }, afterSuccess: ({ context, result }) => { if (result.affectedCount > 0) signalTableRowsChanged(context.tableId) }, @@ -1464,6 +1517,14 @@ export const deleteTableRow = defineAuthorizedTableUseCase({ await deleteRow(context.table, input.rowId, requestId(input)) return { table: context.table, deletedRowId: input.rowId } }, + projectAudit: ({ context, result }) => ({ + action: AuditAction.TABLE_UPDATED, + resourceType: AuditResourceType.TABLE, + resourceId: context.tableId, + resourceName: context.table.name, + description: `Deleted a row from table "${context.table.name}"`, + metadata: { op: 'delete', rowId: result.deletedRowId, rowsDeleted: 1 }, + }), afterSuccess: ({ context, input }) => signalTableRowsChangedByActor(context.tableId, input.actorClientId), }) @@ -1607,5 +1668,18 @@ export const upsertTableRow = defineAuthorizedTableUseCase({ }), } }, + projectAudit: ({ context, result }) => ({ + action: AuditAction.TABLE_UPDATED, + resourceType: AuditResourceType.TABLE, + resourceId: context.tableId, + resourceName: context.table.name, + description: `${result.operation === 'insert' ? 'Inserted' : 'Updated'} a row in table "${context.table.name}"`, + metadata: { + op: 'upsert', + upsertOperation: result.operation, + rowId: result.row.id, + ...(result.operation === 'insert' ? { rowsInserted: 1 } : { rowsUpdated: 1 }), + }, + }), afterSuccess: ({ context }) => signalTableRowsChanged(context.tableId), }) diff --git a/apps/sim/lib/workflows/deployment-outbox.test.ts b/apps/sim/lib/workflows/deployment-outbox.test.ts index 9f3c09ca3de..fb1d932aa5a 100644 --- a/apps/sim/lib/workflows/deployment-outbox.test.ts +++ b/apps/sim/lib/workflows/deployment-outbox.test.ts @@ -120,7 +120,7 @@ import { WORKFLOW_DEPLOYMENT_OUTBOX_EVENTS } from '@/lib/workflows/deployment-ou const mockTx = dbChainMock.db -const mockRecordAudit = auditMockFns.mockRecordAudit +const mockRecordAudit = auditMockFns.mockRecordAuditOnce const mockCaptureServerEvent = posthogServerMockFns.mockCaptureServerEvent @@ -363,6 +363,27 @@ describe('versioned deployment preparation outbox', () => { expect(mockActivateDeploymentOperation).not.toHaveBeenCalled() }) + it('keeps activation audit retryable when its durable insert fails', async () => { + mockIsDeploymentOperationCurrent.mockResolvedValue(true) + mockGetDeploymentOperation.mockResolvedValue(operation({ status: 'active', completedAt: NOW })) + queueTableRows(schemaMock.workflow, [ + { id: 'workflow-1', name: 'Workflow', workspaceId: 'workspace-1' }, + ]) + auditMockFns.mockRecordAuditOnce.mockRejectedValueOnce(new Error('Audit database unavailable')) + const outboxContext = context() + const checkpoints: Record = { inactiveCleanupCompleted: true } + vi.mocked(outboxContext.checkpointPayload).mockImplementation(async (value) => { + if (value && typeof value === 'object' && 'checkpoints' in value) { + Object.assign(checkpoints, value.checkpoints) + } + }) + + await expect( + handler()({ ...payload(), checkpoints: { inactiveCleanupCompleted: true } }, outboxContext) + ).rejects.toThrow('Audit database unavailable') + expect(checkpoints.auditEmitted).toBeUndefined() + }) + /** * Analytics was briefly flushed durably here, which put a deploy's audit * trail, socket notification, and subscription cleanup behind PostHog and diff --git a/apps/sim/lib/workflows/deployment-outbox.ts b/apps/sim/lib/workflows/deployment-outbox.ts index 566fc643351..96c2d3fa7c0 100644 --- a/apps/sim/lib/workflows/deployment-outbox.ts +++ b/apps/sim/lib/workflows/deployment-outbox.ts @@ -1,4 +1,4 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { AuditAction, AuditResourceType, recordAuditOnce } from '@sim/audit' import type { PrincipalActor } from '@sim/auth/principal' import { db, workflowDeploymentVersion, workflow as workflowTable } from '@sim/db' import { outboxEvent, workspaceOperationReceipt } from '@sim/db/schema' @@ -755,7 +755,7 @@ async function emitPostActivationSideEffects(params: { if (!params.checkpoints.auditEmitted) { params.context.signal.throwIfAborted() const isVersionActivation = params.operation.action === 'activate' - recordAudit({ + await recordAuditOnce(`${params.operation.id}:deployment-audit`, { workspaceId: (params.workflow.workspaceId as string) || null, actorId: params.operation.actorId, action: isVersionActivation diff --git a/packages/audit/src/types.ts b/packages/audit/src/types.ts index 8f4102e14e7..e8a56596071 100644 --- a/packages/audit/src/types.ts +++ b/packages/audit/src/types.ts @@ -3,6 +3,13 @@ */ export const AuditAction = { ACCOUNT_DELETED: 'account.deleted', + AUTH_LOGIN_SUCCEEDED: 'auth.login_succeeded', + AUTH_LOGIN_FAILED: 'auth.login_failed', + AUTH_LOGOUT: 'auth.logout', + AUTH_IMPERSONATION_STARTED: 'auth.impersonation_started', + AUTH_IMPERSONATION_ENDED: 'auth.impersonation_ended', + AUTH_SESSIONS_REVOKED: 'auth.sessions_revoked', + AUDIT_LOGS_EXPORTED: 'audit_logs.exported', API_KEY_CREATED: 'api_key.created', API_KEY_UPDATED: 'api_key.updated', @@ -149,6 +156,9 @@ export const AuditAction = { ORGANIZATION_DELETED: 'organization.deleted', ORGANIZATION_SESSION_POLICY_UPDATED: 'organization.session_policy.updated', ORGANIZATION_SSO_POLICY_UPDATED: 'organization.sso_policy.updated', + ORGANIZATION_SSO_PROVIDER_CREATED: 'organization.sso_provider.created', + ORGANIZATION_SSO_PROVIDER_UPDATED: 'organization.sso_provider.updated', + ORGANIZATION_SSO_PROVIDER_DELETED: 'organization.sso_provider.deleted', ORGANIZATION_SESSIONS_REVOKED: 'organization.sessions.revoked', ORGANIZATION_DOMAIN_ADDED: 'organization.domain.added', ORGANIZATION_DOMAIN_VERIFIED: 'organization.domain.verified', diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 94965b05fcc..d3f2f882663 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -7537,7 +7537,7 @@ export const dataDrains = pgTable( destinationCredentials: text('destination_credentials').notNull(), scheduleCadence: dataDrainCadenceEnum('schedule_cadence').notNull(), enabled: boolean('enabled').notNull().default(true), - /** Opaque cursor — JSON-encoded, source-defined. Advances only on overall run success. */ + /** Opaque cursor — JSON-encoded, source-defined. Advances after each acknowledged delivery checkpoint. */ cursor: text('cursor'), lastRunAt: timestamp('last_run_at'), lastSuccessAt: timestamp('last_success_at'), diff --git a/packages/testing/src/mocks/audit.mock.ts b/packages/testing/src/mocks/audit.mock.ts index 2df4fc55132..ecd491c798c 100644 --- a/packages/testing/src/mocks/audit.mock.ts +++ b/packages/testing/src/mocks/audit.mock.ts @@ -7,6 +7,13 @@ import { vi } from 'vitest' */ const AuditAction = { ACCOUNT_DELETED: 'account.deleted', + AUTH_LOGIN_SUCCEEDED: 'auth.login_succeeded', + AUTH_LOGIN_FAILED: 'auth.login_failed', + AUTH_LOGOUT: 'auth.logout', + AUTH_IMPERSONATION_STARTED: 'auth.impersonation_started', + AUTH_IMPERSONATION_ENDED: 'auth.impersonation_ended', + AUTH_SESSIONS_REVOKED: 'auth.sessions_revoked', + AUDIT_LOGS_EXPORTED: 'audit_logs.exported', API_KEY_CREATED: 'api_key.created', API_KEY_UPDATED: 'api_key.updated', API_KEY_REVOKED: 'api_key.revoked', @@ -119,6 +126,9 @@ const AuditAction = { ORGANIZATION_DELETED: 'organization.deleted', ORGANIZATION_SESSION_POLICY_UPDATED: 'organization.session_policy.updated', ORGANIZATION_SSO_POLICY_UPDATED: 'organization.sso_policy.updated', + ORGANIZATION_SSO_PROVIDER_CREATED: 'organization.sso_provider.created', + ORGANIZATION_SSO_PROVIDER_UPDATED: 'organization.sso_provider.updated', + ORGANIZATION_SSO_PROVIDER_DELETED: 'organization.sso_provider.deleted', ORGANIZATION_SESSIONS_REVOKED: 'organization.sessions.revoked', ORGANIZATION_DOMAIN_ADDED: 'organization.domain.added', ORGANIZATION_DOMAIN_VERIFIED: 'organization.domain.verified', diff --git a/vitest.shared.ts b/vitest.shared.ts index f5c24a401f9..14dd9e11f38 100644 --- a/vitest.shared.ts +++ b/vitest.shared.ts @@ -17,6 +17,7 @@ import { BaseSequencer, type TestSpecification } from 'vitest/node' */ export default defineConfig({ test: { + fsModuleCache: true, exclude: [...configDefaults.exclude, '**/dist/**'], clearMocks: true, restoreMocks: true, From 8a2f26b172f3324a13e7fc00c19be01592507c39 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 19:54:23 -0700 Subject: [PATCH 2/3] fix(audit): persist provider acknowledgements and enforce export bounds --- apps/sim/app/api/audit-logs/export/route.ts | 43 +------ .../application/export-audit-logs.ts | 5 +- apps/sim/lib/audit-logs/csv.ts | 41 +++++++ apps/sim/lib/audit-logs/export.integration.ts | 27 +++++ .../lib/auth/lifecycle-audit.integration.ts | 42 +++++++ apps/sim/lib/auth/lifecycle-audit.ts | 29 +++-- .../fixtures/gateway-runtime.fixture.ts | 12 ++ .../lib/core/network/gateway.server.test.ts | 29 ++++- apps/sim/lib/core/network/transport.server.ts | 13 +- .../core/security/input-validation.server.ts | 10 +- .../lib/data-drains/destinations/datadog.ts | 8 +- .../destinations/s3-transport.test.ts | 25 ++++ apps/sim/lib/data-drains/destinations/s3.ts | 3 +- apps/sim/lib/data-drains/enqueue.ts | 44 +++++-- .../lib/data-drains/service.integration.ts | 111 +++++++++++++++--- apps/sim/lib/data-drains/service.ts | 41 +++++-- .../lib/data-drains/sources/copilot-chats.ts | 10 +- .../sources/coverage.integration.ts | 33 ++++++ apps/sim/lib/data-drains/types.ts | 2 + scripts/check-unused-exports.baseline.json | 1 - 20 files changed, 424 insertions(+), 105 deletions(-) create mode 100644 apps/sim/lib/audit-logs/csv.ts diff --git a/apps/sim/app/api/audit-logs/export/route.ts b/apps/sim/app/api/audit-logs/export/route.ts index d559d6c10eb..b8dca5c8470 100644 --- a/apps/sim/app/api/audit-logs/export/route.ts +++ b/apps/sim/app/api/audit-logs/export/route.ts @@ -7,24 +7,7 @@ import { } from '@/lib/api/server/routes' import { exportAuditLogs } from '@/lib/audit-logs/application/export-audit-logs' import { auditLogOperations } from '@/lib/audit-logs/application/operations' -import { formatCsvValue, toCsvRow } from '@/lib/core/utils/csv' - -const CSV_HEADER = toCsvRow([ - 'Date', - 'Action', - 'Resource Type', - 'Resource Name', - 'Actor', - 'Description', - 'Event ID', - 'Workspace ID', - 'Resource ID', - 'Actor ID', - 'IP Address', - 'User Agent', - 'Surface', - 'Metadata', -]) +import { AUDIT_LOG_CSV_HEADER, toAuditLogCsvRow } from '@/lib/audit-logs/csv' export const GET = defineInternalBinaryRoute({ contract: exportAuditLogsContract, @@ -38,29 +21,7 @@ export const GET = defineInternalBinaryRoute({ }, useCase: exportAuditLogs, present: ({ rows, truncated }) => ({ - body: [ - CSV_HEADER, - ...rows.map((row) => - toCsvRow( - [ - row.createdAt, - row.action, - row.resourceType, - row.resourceName, - row.actorEmail || row.actorName || 'System', - row.description, - row.id, - row.workspaceId, - row.resourceId, - row.actorId, - row.ipAddress, - row.userAgent, - row.surface, - row.metadata, - ].map((value) => formatCsvValue(value)) - ) - ), - ].join('\n'), + body: [AUDIT_LOG_CSV_HEADER, ...rows.map(toAuditLogCsvRow)].join('\n'), contentType: 'text/csv; charset=utf-8', contentDisposition: `attachment; filename="audit-logs-${new Date().toISOString().slice(0, 10)}.csv"`, headers: { 'Cache-Control': 'no-store', 'X-Export-Truncated': truncated ? '1' : '0' }, diff --git a/apps/sim/lib/audit-logs/application/export-audit-logs.ts b/apps/sim/lib/audit-logs/application/export-audit-logs.ts index 6a11256110b..5c4edeff41d 100644 --- a/apps/sim/lib/audit-logs/application/export-audit-logs.ts +++ b/apps/sim/lib/audit-logs/application/export-audit-logs.ts @@ -1,6 +1,7 @@ import { AuditAction, AuditResourceType } from '@sim/audit' import { defineAuthorizedAuditLogUseCase } from '@/lib/audit-logs/application/authorized-audit-log-use-case' import { auditLogOperations } from '@/lib/audit-logs/application/operations' +import { AUDIT_LOG_CSV_HEADER, toAuditLogCsvRow } from '@/lib/audit-logs/csv' import { type AuditLogFilterParams, buildFilterConditions, @@ -45,7 +46,7 @@ export const exportAuditLogs = defineAuthorizedAuditLogUseCase({ const conditions = [scope, ...buildFilterConditions(input.filters)] const rows: Awaited>['data'] = [] let cursor: string | undefined - let bytes = 0 + let bytes = Buffer.byteLength(AUDIT_LOG_CSV_HEADER, 'utf8') let truncated = false pages: while (rows.length < EXPORT_MAX_ROWS) { request?.signal?.throwIfAborted() @@ -55,7 +56,7 @@ export const exportAuditLogs = defineAuthorizedAuditLogUseCase({ cursor ) for (const row of page.data) { - bytes += Buffer.byteLength(JSON.stringify(row), 'utf8') + bytes += Buffer.byteLength(toAuditLogCsvRow(row), 'utf8') + 1 if (bytes > EXPORT_MAX_BYTES) { truncated = true break pages diff --git a/apps/sim/lib/audit-logs/csv.ts b/apps/sim/lib/audit-logs/csv.ts new file mode 100644 index 00000000000..36bf129de49 --- /dev/null +++ b/apps/sim/lib/audit-logs/csv.ts @@ -0,0 +1,41 @@ +import type { queryAuditLogs } from '@/lib/audit-logs/query' +import { formatCsvValue, toCsvRow } from '@/lib/core/utils/csv' + +export const AUDIT_LOG_CSV_HEADER = toCsvRow([ + 'Date', + 'Action', + 'Resource Type', + 'Resource Name', + 'Actor', + 'Description', + 'Event ID', + 'Workspace ID', + 'Resource ID', + 'Actor ID', + 'IP Address', + 'User Agent', + 'Surface', + 'Metadata', +]) + +/** Formats the exported row identically for byte admission and the download response. */ +export function toAuditLogCsvRow(row: Awaited>['data'][number]) { + return toCsvRow( + [ + row.createdAt, + row.action, + row.resourceType, + row.resourceName, + row.actorEmail || row.actorName || 'System', + row.description, + row.id, + row.workspaceId, + row.resourceId, + row.actorId, + row.ipAddress, + row.userAgent, + row.surface, + row.metadata, + ].map((value) => formatCsvValue(value)) + ) +} diff --git a/apps/sim/lib/audit-logs/export.integration.ts b/apps/sim/lib/audit-logs/export.integration.ts index ee5e5c70328..9168b29b5c2 100644 --- a/apps/sim/lib/audit-logs/export.integration.ts +++ b/apps/sim/lib/audit-logs/export.integration.ts @@ -117,4 +117,31 @@ describe('Organization audit CSV export in PostgreSQL', () => { ) expect(response.status).toBe(400) }) + + it('bounds encoded CSV bytes and marks quote-heavy metadata as truncated', async () => { + const { db, schema, eq } = runtime + await db + .update(schema.auditLog) + .set({ + metadata: { organizationId, fixture: '"'.repeat(24_000_000) }, + }) + .where(eq(schema.auditLog.id, eventId)) + try { + const response = await runtime.GET( + new NextRequest( + `http://localhost:3000/api/audit-logs/export?organizationId=${organizationId}&includeDeparted=true&actorId=${departedId}` + ) + ) + expect(response.status).toBe(200) + const csv = await response.arrayBuffer() + expect(csv.byteLength).toBeLessThanOrEqual(64 * 1024 * 1024) + expect(response.headers.get('x-export-truncated')).toBe('1') + expect(Buffer.from(csv).toString('utf8')).not.toContain(eventId) + } finally { + await db + .update(schema.auditLog) + .set({ metadata: { organizationId, changedFields: ['name'] } }) + .where(eq(schema.auditLog.id, eventId)) + } + }) }) diff --git a/apps/sim/lib/auth/lifecycle-audit.integration.ts b/apps/sim/lib/auth/lifecycle-audit.integration.ts index 1ec204eeeae..d83df07292d 100644 --- a/apps/sim/lib/auth/lifecycle-audit.integration.ts +++ b/apps/sim/lib/auth/lifecycle-audit.integration.ts @@ -224,6 +224,48 @@ describe('authentication audit history in PostgreSQL', () => { expect((await history()).some((event) => event.action === 'auth.login_succeeded')).toBe(false) }) + it.each([ + { path: '/sign-in/email', action: 'auth.login_failed' }, + { path: '/request-password-reset', action: 'password.reset_requested' }, + ])('omits ambiguous legacy email provenance for $action', async ({ path, action }) => { + const collisionId = generateId() + additionalUserIds.push(collisionId) + await db.insert(user).values({ + id: collisionId, + name: 'Legacy authentication fixture', + email: email.toUpperCase(), + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + }) + await db + .insert(member) + .values({ id: generateId(), userId: collisionId, organizationId, role: 'member' }) + const agent = `Authentication collision fixture ${generateId()}` + const attempt = request(path, { + email, + password: 'incorrect-fixture-password', + redirectTo: 'http://localhost:3000/reset-password', + }) + attempt.headers.set('user-agent', agent) + try { + const response = await auth.handler(attempt) + expect(response.status).toBe(path === '/sign-in/email' ? 401 : 200) + const events = () => db.select().from(auditLog).where(eq(auditLog.userAgent, agent)) + await expect + .poll(async () => (await events()).filter((event) => event.action === action).length) + .toBe(1) + const event = (await events()).find((event) => event.action === action) + if (!event) throw new Error('Authentication audit event missing') + expect(event.actorId).toBeNull() + expect(event.resourceId).toBeNull() + expect(event.metadata).not.toHaveProperty('targetUserId') + expect(event.metadata).not.toHaveProperty('organizationId') + } finally { + await db.delete(auditLog).where(eq(auditLog.userAgent, agent)) + } + }) + it('attributes a password-reset request to its unauthenticated requester and the target organization', async () => { const response = await resetPassword( createMockRequest({ diff --git a/apps/sim/lib/auth/lifecycle-audit.ts b/apps/sim/lib/auth/lifecycle-audit.ts index ef93e747370..a1f015c2074 100644 --- a/apps/sim/lib/auth/lifecycle-audit.ts +++ b/apps/sim/lib/auth/lifecycle-audit.ts @@ -65,18 +65,23 @@ function isFailureRedirect(location: string | null | undefined): boolean { async function loadSubject(input: { userId?: string; email?: unknown }) { const email = typeof input.email === 'string' ? input.email.trim().toLowerCase() : undefined if (!input.userId && !email) return undefined - const [subject] = await db + const subjects = await db .select({ id: user.id, name: user.name, email: user.email, - organizationId: member.organizationId, }) .from(user) - .leftJoin(member, eq(member.userId, user.id)) .where(input.userId ? eq(user.id, input.userId) : eq(foldedEmail(user.email), email ?? '')) + .limit(2) + if (subjects.length !== 1) return undefined + const subject = subjects[0] + const [membership] = await db + .select({ organizationId: member.organizationId }) + .from(member) + .where(eq(member.userId, subject.id)) .limit(1) - return subject + return { ...subject, organizationId: membership?.organizationId } } /** Records final authentication outcomes without persisting request bodies or bearer credentials. */ @@ -178,21 +183,21 @@ export async function recordPasswordResetAudit(input: { }): Promise { try { const subject = await loadSubject(input) - if (!subject) return const completed = input.kind === 'completed' + if (completed && !subject) return recordAudit({ - actorId: completed ? subject.id : null, - actorName: completed ? subject.name : 'Unauthenticated requester', - actorEmail: completed ? subject.email : null, + actorId: completed && subject ? subject.id : null, + actorName: completed && subject ? subject.name : 'Unauthenticated requester', + actorEmail: completed && subject ? subject.email : null, action: completed ? AuditAction.PASSWORD_RESET : AuditAction.PASSWORD_RESET_REQUESTED, resourceType: AuditResourceType.PASSWORD, - resourceId: subject.id, - resourceName: subject.email, + resourceId: subject?.id, + resourceName: subject?.email, description: completed ? 'Password reset completed' : 'Password reset requested', metadata: { outcome: 'success', - ...(subject.organizationId ? { organizationId: subject.organizationId } : {}), - ...(!completed ? { targetUserId: subject.id } : {}), + ...(subject?.organizationId ? { organizationId: subject.organizationId } : {}), + ...(!completed && subject ? { targetUserId: subject.id } : {}), }, request: input.request, }) diff --git a/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts index 7551e2e72ed..cca89aecec0 100644 --- a/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts +++ b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts @@ -35,6 +35,18 @@ try { }) if (pinned.headers.get('x-via-proxy') !== 'yes') throw new Error('Pinned fetch bypassed proxy') if ((await pinned.text()) !== 'tls reached') throw new Error('Pinned fetch response mismatch') + const target = '/exports/../raw/event%2Fpart.ndjson?x-id=PutObject&token=%2B' + const signed = await secureFetchWithPinnedIP(`https://origin.invalid${target}`, '1.1.1.1', { + profile: 'selfHostedService', + requestTarget: target, + method: 'PUT', + body: '{"id":"event"}\n', + timeout: 10_000, + }) + if (signed.headers.get('x-via-proxy') !== 'yes') throw new Error('Signed fetch bypassed proxy') + if (signed.headers.get('x-request-target') !== target) throw new Error('Signed target changed') + if ((await signed.text()) !== '{"id":"event"}\n') + throw new Error('Signed fetch response mismatch') }) let rejected = false diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts index 4a0508112bb..478d86654ab 100644 --- a/apps/sim/lib/core/network/gateway.server.test.ts +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -43,11 +43,14 @@ const admissions: Array<{ }> = [] let originRequests = 0 let receivedHeaders: Record = {} +let receivedTarget = '' const proxiedPorts = new Set() const secureOrigin = httpsServer(async (req, res) => { res.setHeader('x-via-proxy', proxiedPorts.has(req.socket.remotePort ?? 0) ? 'yes' : 'no') originRequests++ receivedHeaders = req.headers + receivedTarget = req.url ?? '' + res.setHeader('x-request-target', receivedTarget) if (req.url === '/redirect') { res.writeHead(302, { location: '/done' }) res.end() @@ -56,7 +59,7 @@ const secureOrigin = httpsServer(async (req, res) => { if (req.url === '/wait') return const chunks: Buffer[] = [] for await (const chunk of req) chunks.push(Buffer.from(chunk)) - res.end(req.method === 'POST' ? Buffer.concat(chunks) : 'tls reached') + res.end(req.method === 'POST' || req.method === 'PUT' ? Buffer.concat(chunks) : 'tls reached') }) const proxy = httpsServer() proxy.on('connect', (req, socket, head) => { @@ -150,6 +153,28 @@ const url = () => 'https://origin.invalid' const options = { profile: 'selfHostedService' as const } describe('organization gateways over real TLS CONNECT sockets', () => { + it('preserves signed request targets and origin authentication through the gateway', async () => { + const target = '/exports/../raw/audit_logs/event%2Fpart.ndjson?x-id=PutObject&token=%2B' + const response = await runWithOutboundOrganization('org_a', () => + secureFetchWithPinnedIP(`${url()}${target}`, '1.1.1.1', { + ...options, + requestTarget: target, + method: 'PUT', + body: '{"id":"event"}\n', + headers: { + authorization: 'AWS4-HMAC-SHA256 signed-request', + 'Proxy-Authorization': 'must-not-reach-origin', + }, + }) + ) + expect(await response.text()).toBe('{"id":"event"}\n') + expect(response.headers.get('x-via-proxy')).toBe('yes') + expect(receivedTarget).toBe(target) + expect(receivedHeaders.authorization).toBe('AWS4-HMAC-SHA256 signed-request') + expect(receivedHeaders['proxy-authorization']).toBeUndefined() + expect(admissions.at(-1)?.destination).toBe('1.1.1.1:443') + }) + it('keeps an upgraded tunnel usable after five minutes', async () => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) let tunnel: Socket | undefined @@ -312,7 +337,7 @@ describe('organization gateways over real TLS CONNECT sockets', () => { }, } ) - expect(admissions.length).toBe(start + 3) + expect(admissions.length).toBe(start + 4) expect( admissions.slice(start).every((entry) => entry.token === `Bearer ${'a'.repeat(48)}`) ).toBe(true) diff --git a/apps/sim/lib/core/network/transport.server.ts b/apps/sim/lib/core/network/transport.server.ts index b4a6c630e7b..d925ac17898 100644 --- a/apps/sim/lib/core/network/transport.server.ts +++ b/apps/sim/lib/core/network/transport.server.ts @@ -79,6 +79,7 @@ type OutboundRequestOptions = Omit< > & { headers?: Record dispatcher: Dispatcher + requestTarget?: string } /** @@ -86,9 +87,19 @@ type OutboundRequestOptions = Omit< * undici shim ignores dispatchers. Proxy credentials never become destination headers. */ export function requestWithOutboundDispatcher(url: string, options: OutboundRequestOptions) { + const { requestTarget, dispatcher, ...requestOptions } = options const headers = { ...options.headers } for (const name of Object.keys(headers)) { if (name.toLowerCase() === 'proxy-authorization') delete headers[name] } - return request(url, { ...options, headers }) + if (requestTarget !== undefined) { + return dispatcher.request({ + ...requestOptions, + origin: new URL(url).origin, + path: requestTarget, + method: requestOptions.method ?? 'GET', + headers, + }) + } + return request(url, { ...requestOptions, dispatcher, headers }) } diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 95fbfcc1259..9941d88e169 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -1156,13 +1156,10 @@ export async function secureFetchWithPinnedIP( proxyUrl: options.proxyUrl, }) const outboundDispatcher = await transport.selectDispatcher() - if (requestTarget !== undefined && outboundDispatcher) { - await transport.destroy() - throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') - } return new Promise((resolve, reject) => { const parsed = new URL(url) + const hostname = unwrapIpv6Brackets(parsed.hostname) const isHttps = parsed.protocol === 'https:' const defaultPort = isHttps ? 443 : 80 const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort @@ -1183,7 +1180,7 @@ export async function secureFetchWithPinnedIP( } else { pinnedLookup = createPinnedLookup(resolvedIP) agent = - options.connectionPool?.agent(isHttps, parsed.hostname, port, resolvedIP) ?? + options.connectionPool?.agent(isHttps, hostname, port, resolvedIP) ?? (isHttps ? new https.Agent({ lookup: pinnedLookup }) : new http.Agent({ lookup: pinnedLookup })) @@ -1207,7 +1204,7 @@ export async function secureFetchWithPinnedIP( } const requestOptions: http.RequestOptions = { - hostname: parsed.hostname, + hostname, port, path: requestTarget ?? parsed.pathname + parsed.search, method: options.method || 'GET', @@ -1519,6 +1516,7 @@ export async function secureFetchWithPinnedIP( send = () => { void requestWithOutboundDispatcher(url, { dispatcher, + requestTarget, method: (options.method || 'GET') as Dispatcher.HttpMethod, headers: sanitizedHeaders, body: options.body, diff --git a/apps/sim/lib/data-drains/destinations/datadog.ts b/apps/sim/lib/data-drains/destinations/datadog.ts index d57568e3e7f..fe74630ad0f 100644 --- a/apps/sim/lib/data-drains/destinations/datadog.ts +++ b/apps/sim/lib/data-drains/destinations/datadog.ts @@ -224,7 +224,7 @@ export const datadogDestination: DrainDestination< openSession({ config, credentials }) { const url = buildEndpoint(config.site) return { - async deliver({ body, metadata, signal }) { + async deliver({ body, metadata, signal, acknowledge }) { const rows = parseNdjsonObjects(body) const entries = buildEntries(rows, config, metadata) const serializedEntries = entries.map((entry, index) => { @@ -252,6 +252,12 @@ export const datadogDestination: DrainDestination< } const response = await postWithRetries({ url, prepared, signal }) requestId = response.headers.get('dd-request-id') + await acknowledge?.({ + rowCount: batch.length, + locator: requestId + ? `datadog://${config.site}#${metadata.runId}-${metadata.sequence}@${requestId}` + : `datadog://${config.site}#${metadata.runId}-${metadata.sequence}`, + }) rawBytes += prepared.rawBytes wireBytes += prepared.wireBytes requests++ diff --git a/apps/sim/lib/data-drains/destinations/s3-transport.test.ts b/apps/sim/lib/data-drains/destinations/s3-transport.test.ts index c2679d8ccf5..0341cb9138f 100644 --- a/apps/sim/lib/data-drains/destinations/s3-transport.test.ts +++ b/apps/sim/lib/data-drains/destinations/s3-transport.test.ts @@ -51,6 +51,31 @@ afterEach(() => { }) describe('S3 drain transport security', () => { + it('delivers signed SDK requests to an already-bracketed public IPv6 endpoint', async () => { + const address = '2606:4700:4700::1111' + vi.spyOn(dns, 'lookup').mockResolvedValue([{ address, family: 6 }]) + const session = s3Destination.openSession({ + config: { ...config, endpoint: `https://[${address}]`, forcePathStyle: true }, + credentials, + }) + try { + await session.deliver({ + body: Buffer.from('{"id":"event"}\n'), + contentType: 'application/x-ndjson', + metadata, + signal: new AbortController().signal, + }) + expect(requests).toHaveLength(1) + expect(requests[0].hostname).toBe(address) + expect(requests[0].headers).toMatchObject({ + host: `[${address}]`, + authorization: expect.stringContaining('AWS4-HMAC-SHA256'), + }) + } finally { + await session.close() + } + }) + it('preserves the signed object request target through the shared egress guard', async () => { const target = '/exports/../raw/audit_logs/event.ndjson?x-id=PutObject' const response = await secureFetchWithPinnedIP( diff --git a/apps/sim/lib/data-drains/destinations/s3.ts b/apps/sim/lib/data-drains/destinations/s3.ts index 11a4daf5c2c..6a3566255db 100644 --- a/apps/sim/lib/data-drains/destinations/s3.ts +++ b/apps/sim/lib/data-drains/destinations/s3.ts @@ -132,7 +132,8 @@ function createRequestHandler(): S3RequestHandler { const signal = callerSignal ? AbortSignal.any([callerSignal, timeout]) : timeout signal.throwIfAborted() const hostname = request.hostname - const authority = hostname.includes(':') ? `[${hostname}]` : hostname + const authority = + hostname.includes(':') && !hostname.startsWith('[') ? `[${hostname}]` : hostname const origin = `https://${authority}${request.port ? `:${request.port}` : ''}` const validation = await validateUrlWithDNS(origin, 'S3 endpoint', 'configuredEndpoint', { signal, diff --git a/apps/sim/lib/data-drains/enqueue.ts b/apps/sim/lib/data-drains/enqueue.ts index dc6aee9c73a..f490b7d374f 100644 --- a/apps/sim/lib/data-drains/enqueue.ts +++ b/apps/sim/lib/data-drains/enqueue.ts @@ -1,31 +1,51 @@ -import { getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs' +import { interruptibleSleep } from '@sim/utils/helpers' +import { generateId } from '@sim/utils/id' +import { backoffWithJitter } from '@sim/utils/retry' +import { getJobQueue, isAsyncJobEnqueueError, shouldExecuteInline } from '@/lib/core/async-jobs' import { DATA_DRAIN_LIMITS } from '@/lib/data-drains/limits' import { runDrain } from '@/lib/data-drains/service' import type { RunTrigger } from '@/lib/data-drains/types' const INLINE_CONCURRENCY_LIMIT = 2 +const MAX_ENQUEUE_ATTEMPTS = 3 /** Runs one bounded export window and schedules a serialized continuation when a backlog remains. */ export async function runQueuedDrain(drainId: string, trigger: RunTrigger, signal: AbortSignal) { const result = await runDrain(drainId, trigger, { signal }) if (result.status === 'success' && result.hasMore) { - await enqueueDrain(drainId, trigger) + await enqueueDrain(drainId, trigger, signal) } return result } /** Bounds inline exports per app process and uses per-drain serialization on isolated external workers. */ -export async function enqueueDrain(drainId: string, trigger: RunTrigger): Promise { +export async function enqueueDrain( + drainId: string, + trigger: RunTrigger, + signal: AbortSignal = new AbortController().signal +): Promise { const queue = await getJobQueue() const inline = shouldExecuteInline() - return queue.enqueue( - 'run-data-drain', - { drainId, trigger }, - { - concurrencyKey: inline ? 'data-drains:inline' : `data-drain:${drainId}`, - concurrencyLimit: inline ? INLINE_CONCURRENCY_LIMIT : 1, - maxDurationSeconds: DATA_DRAIN_LIMITS.hardDurationMs / 1000, - runner: async (_payload, signal) => runQueuedDrain(drainId, trigger, signal), + const jobId = generateId() + for (let attempt = 1; ; attempt++) { + signal.throwIfAborted() + try { + return await queue.enqueue( + 'run-data-drain', + { drainId, trigger }, + { + jobId, + concurrencyKey: inline ? 'data-drains:inline' : `data-drain:${drainId}`, + concurrencyLimit: inline ? INLINE_CONCURRENCY_LIMIT : 1, + maxDurationSeconds: DATA_DRAIN_LIMITS.hardDurationMs / 1000, + runner: async (_payload, signal) => runQueuedDrain(drainId, trigger, signal), + } + ) + } catch (error) { + if (attempt >= MAX_ENQUEUE_ATTEMPTS || !isAsyncJobEnqueueError(error) || !error.retryable) { + throw error + } + await interruptibleSleep(backoffWithJitter(attempt, null), signal) } - ) + } } diff --git a/apps/sim/lib/data-drains/service.integration.ts b/apps/sim/lib/data-drains/service.integration.ts index cdcaa82bdd9..04c59796ccf 100644 --- a/apps/sim/lib/data-drains/service.integration.ts +++ b/apps/sim/lib/data-drains/service.integration.ts @@ -1,5 +1,6 @@ import { mkdir, writeFile } from 'node:fs/promises' import path from 'node:path' +import { gunzipSync } from 'node:zlib' import { db } from '@sim/db' import { asyncJobs, @@ -27,7 +28,10 @@ vi.mock('@/lib/data-drains/destinations/registry', () => ({ getDestination: mock vi.mock('@/lib/billing/core/subscription', () => billingSubscriptionMock) vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) +import { getJobQueue } from '@/lib/core/async-jobs' +import { AsyncJobEnqueueError } from '@/lib/core/async-jobs/types' import { testDataDrain } from '@/lib/data-drains/application/use-cases' +import { datadogDestination } from '@/lib/data-drains/destinations/datadog' import { dispatchDueDrains } from '@/lib/data-drains/dispatcher' import { encryptCredentials } from '@/lib/data-drains/encryption' import { enqueueDrain } from '@/lib/data-drains/enqueue' @@ -161,6 +165,56 @@ afterAll(async () => { }) describe('data drain durable delivery', () => { + it('checkpoints each accepted Datadog request before a later request fails', async () => { + await seedRows(6, 'x'.repeat(425_000)) + await db + .update(dataDrains) + .set({ + destinationType: 'datadog', + destinationConfig: { site: 'us1' }, + destinationCredentials: await encryptCredentials({ apiKey: 'fixture-dd-key' }), + }) + .where(eq(dataDrains.id, drainId)) + mockGetDestination.mockReturnValue(datadogDestination) + const acceptedIds: string[] = [] + const requestIds: string[] = [] + let posts = 0 + vi.stubGlobal('fetch', async (_url: unknown, init?: RequestInit) => { + posts++ + if (posts === 2) return new Response('Rejected fixture request', { status: 403 }) + const body = init?.body + if (typeof body !== 'string' && !(body instanceof Uint8Array)) + throw new Error('Missing Datadog body') + const payload = + new Headers(init?.headers).get('content-encoding') === 'gzip' + ? gunzipSync(body).toString('utf8') + : typeof body === 'string' + ? body + : Buffer.from(body).toString('utf8') + const rows = JSON.parse(payload) as Array<{ id: string }> + acceptedIds.push(...rows.map((row) => row.id)) + const requestId = `fixture-request-${posts}` + requestIds.push(requestId) + return new Response(null, { status: 202, headers: { 'dd-request-id': requestId } }) + }) + try { + await expect(runDrain(drainId, 'cron')).rejects.toThrow(/HTTP 403/) + const checkpoint = (await drainRow()).cursor + expect(checkpoint).not.toBeNull() + expect(JSON.parse(checkpoint ?? '{}').id).toBe(acceptedIds.at(-1)) + await runDrain(drainId, 'cron') + expect(acceptedIds).toEqual( + Array.from({ length: 6 }, (_, index) => `${drainId}-${index.toString().padStart(6, '0')}`) + ) + const runs = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + const locators = runs.flatMap((run) => run.locators ?? []) + for (const id of requestIds) + expect(locators.some((locator) => locator.endsWith(`@${id}`))).toBe(true) + } finally { + vi.unstubAllGlobals() + } + }) + it('fails closed when malformed JSON credentials expose only a private-key fragment', async () => { await seedRows(1) const fragment = 'malformed-private-key-fragment-028f' @@ -466,20 +520,49 @@ describe('data drain durable delivery', () => { expect(received.flat()).toHaveLength(3) }) - it('continues a queued backlog until every bounded window is delivered', async () => { - await seedRows(10001) - await dispatchDueDrains() - await vi.waitFor( - async () => { - expect(received.flat()).toHaveLength(10001) - const runs = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) - expect(runs.length).toBeGreaterThan(1) - expect(runs.every((run) => run.status === 'success')).toBe(true) - }, - { timeout: 10_000 } - ) - expect(new Set(received.flat().map((row) => row.id)).size).toBe(10001) - }, 30_000) + it.each([ + { failContinuation: false, condition: 'without an enqueue failure' }, + { failContinuation: true, condition: 'after a transient continuation enqueue failure' }, + ])( + 'continues every bounded window $condition', + async ({ failContinuation }) => { + await seedRows(10001) + const queue = await getJobQueue() + const enqueue = queue.enqueue.bind(queue) + let rejected = false + const fault = vi + .spyOn(queue, 'enqueue') + .mockImplementation(async (type, payload, options) => { + if (failContinuation && !rejected && received.flat().length === 10000) { + rejected = true + throw new AsyncJobEnqueueError('Transient fixture enqueue rejection', { + acceptance: 'rejected', + retryable: true, + }) + } + return enqueue(type, payload, options) + }) + try { + await dispatchDueDrains() + await vi.waitFor( + async () => { + expect(received.flat()).toHaveLength(10001) + const runs = await db + .select() + .from(dataDrainRuns) + .where(eq(dataDrainRuns.drainId, drainId)) + expect(runs.length).toBeGreaterThan(1) + expect(runs.every((run) => run.status === 'success')).toBe(true) + }, + { timeout: 10_000 } + ) + expect(new Set(received.flat().map((row) => row.id)).size).toBe(10001) + } finally { + fault.mockRestore() + } + }, + 30_000 + ) it('rechecks an organization entitlement before a queued worker reads or delivers its data', async () => { await seedRows(1) diff --git a/apps/sim/lib/data-drains/service.ts b/apps/sim/lib/data-drains/service.ts index b3ae9db724a..9b37302442f 100644 --- a/apps/sim/lib/data-drains/service.ts +++ b/apps/sim/lib/data-drains/service.ts @@ -30,8 +30,8 @@ export interface RunDrainResult { /** * Orchestrates one drain export. Source-/destination-agnostic — talks only to - * the registry interfaces. Each acknowledged chunk checkpoints its cursor; - * a crash between delivery and checkpoint can replay that chunk, so consumers + * the registry interfaces. Each provider acknowledgement checkpoints its cursor; + * a crash between delivery and checkpoint can replay those rows, so consumers * dedupe on the per-row `id` field. A database claim fences concurrent workers. */ export async function runDrain( @@ -195,8 +195,9 @@ export async function runDrain( session = activeSession let lines: string[] = [] + let rowCursors: Cursor[] = [] + let lineSizes: number[] = [] let chunkBytes = 0 - let chunkCursor: Cursor = cursor const softDeadline = startedAt.getTime() + DATA_DRAIN_LIMITS.softDurationMs async function deliverChunk(): Promise { @@ -204,6 +205,25 @@ export async function runDrain( signal.throwIfAborted() const rowCount = lines.length const body = Buffer.from(`${lines.join('\n')}\n`, 'utf8') + let acknowledgedRows = 0 + const acknowledge = async (result: { locator: string; rowCount: number }) => { + if ( + !Number.isInteger(result.rowCount) || + result.rowCount <= 0 || + acknowledgedRows + result.rowCount > rowCount + ) { + throw new Error('Invalid data drain delivery acknowledgement') + } + const nextRows = acknowledgedRows + result.rowCount + locators.push(result.locator) + rowsExported += result.rowCount + for (let index = acknowledgedRows; index < nextRows; index++) { + bytesWritten += lineSizes[index] + } + cursor = rowCursors[nextRows - 1] + await checkpoint('running') + acknowledgedRows = nextRows + } const result = await runWithOutboundOrganization(drain.organizationId, () => activeSession.deliver({ body, @@ -217,15 +237,17 @@ export async function runDrain( runStartedAt: startedAt, }, signal, + acknowledge, }) ) - locators.push(result.locator) - rowsExported += rowCount - bytesWritten += body.byteLength - cursor = chunkCursor + if (acknowledgedRows === 0) await acknowledge({ locator: result.locator, rowCount }) + if (acknowledgedRows !== rowCount) { + throw new Error('Incomplete data drain delivery acknowledgement') + } sequence++ - await checkpoint('running') lines = [] + rowCursors = [] + lineSizes = [] chunkBytes = 0 } @@ -271,8 +293,9 @@ export async function runDrain( break pages } lines.push(line) + rowCursors.push(source.cursorAfter(row)) + lineSizes.push(lineBytes) chunkBytes += lineBytes - chunkCursor = source.cursorAfter(row) } await deliverChunk() if (runWindowFull()) { diff --git a/apps/sim/lib/data-drains/sources/copilot-chats.ts b/apps/sim/lib/data-drains/sources/copilot-chats.ts index da2ccc7409f..b9e7a0b0f26 100644 --- a/apps/sim/lib/data-drains/sources/copilot-chats.ts +++ b/apps/sim/lib/data-drains/sources/copilot-chats.ts @@ -107,11 +107,15 @@ async function* pages(input: SourcePageInput): AsyncIterable { .from(copilotMessages) .where(messageCondition) .groupBy(copilotMessages.chatId) - for (const transcript of transcriptSizes) { + const transcriptBytes = new Map( + transcriptSizes.map((transcript) => [transcript.chatId, transcript.bytes]) + ) + for (const row of metaRows) { assertKnownSizeWithinLimit( - transcript.bytes, + Buffer.byteLength(JSON.stringify({ ...row, messages: [] }), 'utf8') + + (transcriptBytes.get(row.id) ?? 0), DATA_DRAIN_LIMITS.maxRowBytes, - 'Drain transcript record' + 'Drain chat record' ) } const messageRows = await tx diff --git a/apps/sim/lib/data-drains/sources/coverage.integration.ts b/apps/sim/lib/data-drains/sources/coverage.integration.ts index 9ff1afde36e..e22770f9cd5 100644 --- a/apps/sim/lib/data-drains/sources/coverage.integration.ts +++ b/apps/sim/lib/data-drains/sources/coverage.integration.ts @@ -146,6 +146,39 @@ describe('Drain source coverage and payload safety in PostgreSQL', () => { } }) + it('refuses metadata and transcript that together exceed the record budget', async () => { + const { db, schema, eq } = runtime + const [original] = await db + .select() + .from(schema.copilotChats) + .where(eq(schema.copilotChats.id, chatId)) + const messageId = generateId() + await db + .update(schema.copilotChats) + .set({ config: { fixture: 'x'.repeat(550_000) } }) + .where(eq(schema.copilotChats.id, chatId)) + await db.insert(schema.copilotMessages).values({ + id: messageId, + messageId: generateId(), + chatId, + role: 'user', + content: { text: 'x'.repeat(550_000) }, + createdAt: old, + }) + try { + await expect(async () => { + for await (const _rows of runtime.copilotChatsSource.pages(input())) { + } + }).rejects.toThrow(/record.*(bytes|size|limit)/i) + } finally { + await db.delete(schema.copilotMessages).where(eq(schema.copilotMessages.id, messageId)) + await db + .update(schema.copilotChats) + .set({ config: original.config }) + .where(eq(schema.copilotChats.id, chatId)) + } + }) + it('keeps paging when a caller requests more rows than the source page budget', async () => { const scope = generateId() const eventIds = Array.from({ length: 101 }, () => generateId()) diff --git a/apps/sim/lib/data-drains/types.ts b/apps/sim/lib/data-drains/types.ts index 9a0464ce5ce..bb29195282a 100644 --- a/apps/sim/lib/data-drains/types.ts +++ b/apps/sim/lib/data-drains/types.ts @@ -84,6 +84,8 @@ interface DrainDeliverySession { contentType: 'application/x-ndjson' metadata: DeliveryMetadata signal: AbortSignal + /** Checkpoints the next rowCount rows after a provider accepts them, before sending more. */ + acknowledge?: (result: DeliveryResult & { rowCount: number }) => Promise }): Promise close(): Promise } diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index c189a1f719f..9f9755c6ad1 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -4343,7 +4343,6 @@ "apps/sim/lib/core/async-jobs/index.ts#JOB_RETENTION_SECONDS", "apps/sim/lib/core/async-jobs/index.ts#getAsyncBackendType", "apps/sim/lib/core/async-jobs/index.ts#getCurrentBackendType", - "apps/sim/lib/core/async-jobs/index.ts#isAsyncJobEnqueueError", "apps/sim/lib/core/async-jobs/index.ts#resetJobQueueCache", "apps/sim/lib/core/async-jobs/types.ts#AsyncExecutionCorrelationSource", "apps/sim/lib/core/config/env-flags.ts#isEnterpriseEnabled", From a6cc8a8bab2bdbd9874e6ed5bbcc9e1306af618f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 7 Oct 2026 20:19:35 -0700 Subject: [PATCH 3/3] fix(audit): bound CSV allocation and distinguish delivery batches --- .../docs/platform/enterprise/audit-logs.mdx | 2 +- apps/sim/app/api/audit-logs/export/route.ts | 5 +- .../application/export-audit-logs.ts | 28 +++-- apps/sim/lib/audit-logs/csv.ts | 55 +++++---- apps/sim/lib/audit-logs/export.integration.ts | 50 +++++++- apps/sim/lib/audit-logs/query.integration.ts | 44 ++++--- apps/sim/lib/audit-logs/query.ts | 58 +++++++++- .../lib/data-drains/destinations/datadog.ts | 16 +-- .../lib/data-drains/service.integration.ts | 109 ++++++++++-------- 9 files changed, 255 insertions(+), 112 deletions(-) diff --git a/apps/docs/content/docs/platform/enterprise/audit-logs.mdx b/apps/docs/content/docs/platform/enterprise/audit-logs.mdx index f30016c0e55..39c7a63ddb6 100644 --- a/apps/docs/content/docs/platform/enterprise/audit-logs.mdx +++ b/apps/docs/content/docs/platform/enterprise/audit-logs.mdx @@ -128,7 +128,7 @@ Workspace invitation events include whether the invite is for an internal organi }, { question: "Can I export audit logs?", - answer: "Yes. Download CSV from the audit viewer or paginate the API for your SIEM. CSV includes event, workspace, resource and actor IDs, IP address, user agent, surface and metadata. Organization administrators can include departed actors for historical investigations. CSV downloads are recorded in the audit trail and capped at 10,000 rows or 64 MB; the X-Export-Truncated header indicates a partial export. For continuous delivery, use an audit-log data drain." + answer: "Yes. Download CSV from the audit viewer or paginate the API for your SIEM. CSV includes event, workspace, resource and actor IDs, IP address, user agent, surface and metadata. Organization administrators can include departed actors for historical investigations. CSV downloads are recorded in the audit trail and capped at 10,000 rows or 64 MiB. Oversized records also end the download early; the X-Export-Truncated header indicates a partial export. For continuous delivery, use an audit-log data drain." }, { question: "Are logs scoped to a single workspace or the whole organization?", diff --git a/apps/sim/app/api/audit-logs/export/route.ts b/apps/sim/app/api/audit-logs/export/route.ts index b8dca5c8470..d417af06d6d 100644 --- a/apps/sim/app/api/audit-logs/export/route.ts +++ b/apps/sim/app/api/audit-logs/export/route.ts @@ -7,7 +7,6 @@ import { } from '@/lib/api/server/routes' import { exportAuditLogs } from '@/lib/audit-logs/application/export-audit-logs' import { auditLogOperations } from '@/lib/audit-logs/application/operations' -import { AUDIT_LOG_CSV_HEADER, toAuditLogCsvRow } from '@/lib/audit-logs/csv' export const GET = defineInternalBinaryRoute({ contract: exportAuditLogsContract, @@ -20,8 +19,8 @@ export const GET = defineInternalBinaryRoute({ return { organizationId, includeDeparted, filters } }, useCase: exportAuditLogs, - present: ({ rows, truncated }) => ({ - body: [AUDIT_LOG_CSV_HEADER, ...rows.map(toAuditLogCsvRow)].join('\n'), + present: ({ csv, truncated }) => ({ + body: csv, contentType: 'text/csv; charset=utf-8', contentDisposition: `attachment; filename="audit-logs-${new Date().toISOString().slice(0, 10)}.csv"`, headers: { 'Cache-Control': 'no-store', 'X-Export-Truncated': truncated ? '1' : '0' }, diff --git a/apps/sim/lib/audit-logs/application/export-audit-logs.ts b/apps/sim/lib/audit-logs/application/export-audit-logs.ts index 5c4edeff41d..ce0507f2989 100644 --- a/apps/sim/lib/audit-logs/application/export-audit-logs.ts +++ b/apps/sim/lib/audit-logs/application/export-audit-logs.ts @@ -44,30 +44,38 @@ export const exportAuditLogs = defineAuthorizedAuditLogUseCase({ includeDeparted: input.includeDeparted, }) const conditions = [scope, ...buildFilterConditions(input.filters)] - const rows: Awaited>['data'] = [] + const lines = [AUDIT_LOG_CSV_HEADER] + let rowCount = 0 let cursor: string | undefined let bytes = Buffer.byteLength(AUDIT_LOG_CSV_HEADER, 'utf8') let truncated = false - pages: while (rows.length < EXPORT_MAX_ROWS) { + pages: while (rowCount < EXPORT_MAX_ROWS && bytes < EXPORT_MAX_BYTES) { request?.signal?.throwIfAborted() const page = await queryAuditLogs( conditions, - Math.min(EXPORT_PAGE_ROWS, EXPORT_MAX_ROWS - rows.length), - cursor + Math.min(EXPORT_PAGE_ROWS, EXPORT_MAX_ROWS - rowCount), + cursor, + EXPORT_MAX_BYTES ) for (const row of page.data) { - bytes += Buffer.byteLength(toAuditLogCsvRow(row), 'utf8') + 1 - if (bytes > EXPORT_MAX_BYTES) { + const line = toAuditLogCsvRow(row, EXPORT_MAX_BYTES - bytes - 1) + if (line === undefined) { truncated = true break pages } - rows.push(row) + bytes += Buffer.byteLength(line, 'utf8') + 1 + lines.push(line) + rowCount++ + } + if (page.truncated) { + truncated = true + break } if (!page.nextCursor) break cursor = page.nextCursor - truncated = rows.length >= EXPORT_MAX_ROWS + truncated = rowCount >= EXPORT_MAX_ROWS || bytes >= EXPORT_MAX_BYTES } - return { rows, truncated } + return { csv: lines.join('\n'), rowCount, truncated } }, projectAudit: ({ input, result }) => ({ action: AuditAction.AUDIT_LOGS_EXPORTED, @@ -76,7 +84,7 @@ export const exportAuditLogs = defineAuthorizedAuditLogUseCase({ description: 'Exported organization audit logs', metadata: { format: 'csv', - rowCount: result.rows.length, + rowCount: result.rowCount, truncated: result.truncated, includeDeparted: input.includeDeparted, filters: Object.keys(input.filters).filter((key) => diff --git a/apps/sim/lib/audit-logs/csv.ts b/apps/sim/lib/audit-logs/csv.ts index 36bf129de49..4cacba8a852 100644 --- a/apps/sim/lib/audit-logs/csv.ts +++ b/apps/sim/lib/audit-logs/csv.ts @@ -18,24 +18,39 @@ export const AUDIT_LOG_CSV_HEADER = toCsvRow([ 'Metadata', ]) -/** Formats the exported row identically for byte admission and the download response. */ -export function toAuditLogCsvRow(row: Awaited>['data'][number]) { - return toCsvRow( - [ - row.createdAt, - row.action, - row.resourceType, - row.resourceName, - row.actorEmail || row.actorName || 'System', - row.description, - row.id, - row.workspaceId, - row.resourceId, - row.actorId, - row.ipAddress, - row.userAgent, - row.surface, - row.metadata, - ].map((value) => formatCsvValue(value)) - ) +/** Counts CSV escaping before allocating the encoded row, whose database input is already bounded. */ +export function toAuditLogCsvRow( + row: Awaited>['data'][number], + maxBytes: number +): string | undefined { + const values = [ + row.createdAt, + row.action, + row.resourceType, + row.resourceName, + row.actorEmail || row.actorName || 'System', + row.description, + row.id, + row.workspaceId, + row.resourceId, + row.actorId, + row.ipAddress, + row.userAgent, + row.surface, + row.metadata, + ].map((value) => formatCsvValue(value)) + let bytes = values.length - 1 + for (const value of values) { + bytes += Buffer.byteLength(value, 'utf8') + if (bytes > maxBytes) return undefined + if (/[",\n\r]/.test(value)) { + bytes += 2 + for (let quote = value.indexOf('"'); quote !== -1; quote = value.indexOf('"', quote + 1)) { + bytes++ + if (bytes > maxBytes) return undefined + } + } + if (bytes > maxBytes) return undefined + } + return toCsvRow(values) } diff --git a/apps/sim/lib/audit-logs/export.integration.ts b/apps/sim/lib/audit-logs/export.integration.ts index 9168b29b5c2..f944433878e 100644 --- a/apps/sim/lib/audit-logs/export.integration.ts +++ b/apps/sim/lib/audit-logs/export.integration.ts @@ -8,13 +8,13 @@ vi.mock('@/lib/auth', () => authMock) vi.mock('@/lib/core/config/env-flags', () => envFlagsMock) async function loadRuntime() { - const [{ db }, schema, { eq, inArray }, { GET }] = await Promise.all([ + const [{ db }, schema, { eq, inArray, sql }, { GET }] = await Promise.all([ import('@sim/db'), import('@sim/db/schema'), import('drizzle-orm'), import('@/app/api/audit-logs/export/route'), ]) - return { db, schema, eq, inArray, GET } + return { db, schema, eq, inArray, sql, GET } } describe('Organization audit CSV export in PostgreSQL', () => { @@ -144,4 +144,50 @@ describe('Organization audit CSV export in PostgreSQL', () => { .where(eq(schema.auditLog.id, eventId)) } }) + + it('keeps the valid prefix without decoding an oversized database record', async () => { + const { db, schema, eq, sql } = runtime + const prefixId = generateId() + await db.insert(schema.auditLog).values({ + id: prefixId, + actorId: departedId, + action: 'organization.updated', + resourceType: 'organization', + resourceId: organizationId, + metadata: { organizationId }, + createdAt: new Date(Date.now() + 1000), + }) + await db + .update(schema.auditLog) + .set({ + metadata: sql`jsonb_build_object('organizationId', ${organizationId}::text, 'fixture', repeat('x', 70 * 1024 * 1024))`, + }) + .where(eq(schema.auditLog.id, eventId)) + const parse = JSON.parse + let largestDecodedValue = 0 + const decoding = vi.spyOn(JSON, 'parse').mockImplementation((text, reviver) => { + largestDecodedValue = Math.max(largestDecodedValue, Buffer.byteLength(text, 'utf8')) + return parse(text, reviver) + }) + try { + const response = await runtime.GET( + new NextRequest( + `http://localhost:3000/api/audit-logs/export?organizationId=${organizationId}&includeDeparted=true&actorId=${departedId}` + ) + ) + expect(response.status).toBe(200) + expect(response.headers.get('x-export-truncated')).toBe('1') + const csv = await response.text() + expect(csv).toContain(prefixId) + expect(csv).not.toContain(eventId) + expect(largestDecodedValue).toBeLessThanOrEqual(64 * 1024 * 1024) + } finally { + decoding.mockRestore() + await db.delete(schema.auditLog).where(eq(schema.auditLog.id, prefixId)) + await db + .update(schema.auditLog) + .set({ metadata: { organizationId, changedFields: ['name'] } }) + .where(eq(schema.auditLog.id, eventId)) + } + }) }) diff --git a/apps/sim/lib/audit-logs/query.integration.ts b/apps/sim/lib/audit-logs/query.integration.ts index 4f24933206c..4e995665bbd 100644 --- a/apps/sim/lib/audit-logs/query.integration.ts +++ b/apps/sim/lib/audit-logs/query.integration.ts @@ -28,7 +28,8 @@ describe('Audit pagination and drain tenant coverage in PostgreSQL', () => { action: 'organization.updated', resourceType: 'organization', resourceId: index === 3 ? generateId() : organizationId, - metadata: index === 0 ? { organizationId } : {}, + metadata: + index === 0 ? { organizationId, fixture: 'x'.repeat(400) } : { fixture: 'x'.repeat(400) }, createdAt: new Date('2026-01-01T12:00:00.123Z'), })) ) @@ -48,22 +49,33 @@ describe('Audit pagination and drain tenant coverage in PostgreSQL', () => { } }) - it('returns every event once when pages split within a PostgreSQL millisecond', async () => { - const seen: string[] = [] - let cursor: string | undefined - for (let pageIndex = 0; pageIndex < ids.length + 1; pageIndex++) { - const page = await runtime.query.queryAuditLogs( - [runtime.inArray(runtime.auditLog.id, ids)], - 1, - cursor - ) - seen.push(...page.data.map((row) => row.id)) - if (!page.nextCursor) break - cursor = page.nextCursor + it.each([ + { limit: 1, maxBytes: undefined, boundary: 'row limit' }, + { limit: 10, maxBytes: 1000, boundary: 'byte limit' }, + ])( + 'returns every event once across a $boundary within one PostgreSQL millisecond', + async ({ limit, maxBytes }) => { + const seen: string[] = [] + let cursor: string | undefined + for (let pageIndex = 0; pageIndex < ids.length + 1; pageIndex++) { + const page = await runtime.query.queryAuditLogs( + [runtime.inArray(runtime.auditLog.id, ids)], + limit, + cursor, + maxBytes + ) + if (maxBytes !== undefined) { + expect(Buffer.byteLength(JSON.stringify(page.data), 'utf8')).toBeLessThanOrEqual(maxBytes) + expect(page.truncated).toBe(false) + } + seen.push(...page.data.map((row) => row.id)) + if (!page.nextCursor) break + cursor = page.nextCursor + } + expect(seen).toHaveLength(ids.length) + expect(new Set(seen)).toEqual(new Set(ids)) } - expect(seen).toHaveLength(ids.length) - expect(new Set(seen)).toEqual(new Set(ids)) - }) + ) it('drains all organization-resource events and excludes the other tenant', async () => { const seen: string[] = [] diff --git a/apps/sim/lib/audit-logs/query.ts b/apps/sim/lib/audit-logs/query.ts index a31bf0ff667..25f4c92ddc4 100644 --- a/apps/sim/lib/audit-logs/query.ts +++ b/apps/sim/lib/audit-logs/query.ts @@ -158,12 +158,15 @@ function buildCursorCondition(cursor: string): SQL | null { interface CursorPaginatedResult { data: DbAuditLog[] nextCursor?: string + truncated?: boolean } +/** An optional byte budget bounds each database page before hydration and stops at oversized rows. */ export async function queryAuditLogs( conditions: SQL[], limit: number, - cursor?: string + cursor?: string, + maxBytes?: number ): Promise { const allConditions = [...conditions] if (cursor) { @@ -171,11 +174,60 @@ export async function queryAuditLogs( if (cursorCondition) allConditions.push(cursorCondition) } + const condition = allConditions.length > 0 ? and(...allConditions) : undefined + const orderBy = [desc(sql`date_trunc('milliseconds', ${auditLog.createdAt})`), desc(auditLog.id)] + if (maxBytes !== undefined) { + return dbReplica.transaction( + async (tx) => { + const candidates = await tx + .select({ + id: auditLog.id, + bytes: sql`octet_length(row_to_json(${auditLog})::text)`.mapWith(Number), + }) + .from(auditLog) + .where(condition) + .orderBy(...orderBy) + .limit(limit + 1) + const ids: string[] = [] + let bytes = 0 + let truncated = false + for (const candidate of candidates.slice(0, limit)) { + if (candidate.bytes > maxBytes) { + truncated = true + break + } + if (bytes + candidate.bytes > maxBytes) break + ids.push(candidate.id) + bytes += candidate.bytes + } + const data = + ids.length > 0 + ? await tx + .select() + .from(auditLog) + .where(inArray(auditLog.id, ids)) + .orderBy(...orderBy) + .limit(limit) + : [] + const last = data.at(-1) + return { + data, + truncated, + nextCursor: + candidates.length > data.length && last + ? encodeCursor({ createdAt: last.createdAt.toISOString(), id: last.id }) + : undefined, + } + }, + { isolationLevel: 'repeatable read', accessMode: 'read only' } + ) + } + const rows = await dbReplica .select() .from(auditLog) - .where(allConditions.length > 0 ? and(...allConditions) : undefined) - .orderBy(desc(sql`date_trunc('milliseconds', ${auditLog.createdAt})`), desc(auditLog.id)) + .where(condition) + .orderBy(...orderBy) .limit(limit + 1) const hasMore = rows.length > limit diff --git a/apps/sim/lib/data-drains/destinations/datadog.ts b/apps/sim/lib/data-drains/destinations/datadog.ts index fe74630ad0f..edb89210d12 100644 --- a/apps/sim/lib/data-drains/destinations/datadog.ts +++ b/apps/sim/lib/data-drains/destinations/datadog.ts @@ -241,7 +241,8 @@ export const datadogDestination: DrainDestination< let batchBytes = 2 let rawBytes = 0 let wireBytes = 0 - let requestId: string | null = null + const locatorPrefix = `datadog://${config.site}#${metadata.runId}-${metadata.sequence}` + let locator = locatorPrefix let requests = 0 const sendBatch = async () => { const prepared = buildRequestBody(`[${batch.join(',')}]`, credentials.apiKey) @@ -251,12 +252,11 @@ export const datadogDestination: DrainDestination< ) } const response = await postWithRetries({ url, prepared, signal }) - requestId = response.headers.get('dd-request-id') + const requestId = response.headers.get('dd-request-id') + locator = requestId ? `${locatorPrefix}@${requestId}` : `${locatorPrefix}-${requests}` await acknowledge?.({ rowCount: batch.length, - locator: requestId - ? `datadog://${config.site}#${metadata.runId}-${metadata.sequence}@${requestId}` - : `datadog://${config.site}#${metadata.runId}-${metadata.sequence}`, + locator, }) rawBytes += prepared.rawBytes wireBytes += prepared.wireBytes @@ -283,11 +283,7 @@ export const datadogDestination: DrainDestination< rawBytes, wireBytes, }) - return { - locator: requestId - ? `datadog://${config.site}#${metadata.runId}-${metadata.sequence}@${requestId}` - : `datadog://${config.site}#${metadata.runId}-${metadata.sequence}`, - } + return { locator } }, async close() {}, } diff --git a/apps/sim/lib/data-drains/service.integration.ts b/apps/sim/lib/data-drains/service.integration.ts index 04c59796ccf..500d57296a8 100644 --- a/apps/sim/lib/data-drains/service.integration.ts +++ b/apps/sim/lib/data-drains/service.integration.ts @@ -165,55 +165,70 @@ afterAll(async () => { }) describe('data drain durable delivery', () => { - it('checkpoints each accepted Datadog request before a later request fails', async () => { - await seedRows(6, 'x'.repeat(425_000)) - await db - .update(dataDrains) - .set({ - destinationType: 'datadog', - destinationConfig: { site: 'us1' }, - destinationCredentials: await encryptCredentials({ apiKey: 'fixture-dd-key' }), + it.each([ + { requestIdsReturned: true, failSecond: true, condition: 'before a later request fails' }, + { requestIdsReturned: false, failSecond: false, condition: 'without provider request IDs' }, + ])( + 'retains each accepted Datadog request $condition', + async ({ requestIdsReturned, failSecond }) => { + await seedRows(6, 'x'.repeat(425_000)) + await db + .update(dataDrains) + .set({ + destinationType: 'datadog', + destinationConfig: { site: 'us1' }, + destinationCredentials: await encryptCredentials({ apiKey: 'fixture-dd-key' }), + }) + .where(eq(dataDrains.id, drainId)) + mockGetDestination.mockReturnValue(datadogDestination) + const acceptedIds: string[] = [] + const requestIds: string[] = [] + let posts = 0 + vi.stubGlobal('fetch', async (_url: unknown, init?: RequestInit) => { + posts++ + if (failSecond && posts === 2) + return new Response('Rejected fixture request', { status: 403 }) + const body = init?.body + if (typeof body !== 'string' && !(body instanceof Uint8Array)) + throw new Error('Missing Datadog body') + const payload = + new Headers(init?.headers).get('content-encoding') === 'gzip' + ? gunzipSync(body).toString('utf8') + : typeof body === 'string' + ? body + : Buffer.from(body).toString('utf8') + const rows = JSON.parse(payload) as Array<{ id: string }> + acceptedIds.push(...rows.map((row) => row.id)) + const requestId = `fixture-request-${posts}` + requestIds.push(requestId) + return new Response(null, { + status: 202, + headers: requestIdsReturned ? { 'dd-request-id': requestId } : {}, + }) }) - .where(eq(dataDrains.id, drainId)) - mockGetDestination.mockReturnValue(datadogDestination) - const acceptedIds: string[] = [] - const requestIds: string[] = [] - let posts = 0 - vi.stubGlobal('fetch', async (_url: unknown, init?: RequestInit) => { - posts++ - if (posts === 2) return new Response('Rejected fixture request', { status: 403 }) - const body = init?.body - if (typeof body !== 'string' && !(body instanceof Uint8Array)) - throw new Error('Missing Datadog body') - const payload = - new Headers(init?.headers).get('content-encoding') === 'gzip' - ? gunzipSync(body).toString('utf8') - : typeof body === 'string' - ? body - : Buffer.from(body).toString('utf8') - const rows = JSON.parse(payload) as Array<{ id: string }> - acceptedIds.push(...rows.map((row) => row.id)) - const requestId = `fixture-request-${posts}` - requestIds.push(requestId) - return new Response(null, { status: 202, headers: { 'dd-request-id': requestId } }) - }) - try { - await expect(runDrain(drainId, 'cron')).rejects.toThrow(/HTTP 403/) - const checkpoint = (await drainRow()).cursor - expect(checkpoint).not.toBeNull() - expect(JSON.parse(checkpoint ?? '{}').id).toBe(acceptedIds.at(-1)) - await runDrain(drainId, 'cron') - expect(acceptedIds).toEqual( - Array.from({ length: 6 }, (_, index) => `${drainId}-${index.toString().padStart(6, '0')}`) - ) - const runs = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) - const locators = runs.flatMap((run) => run.locators ?? []) - for (const id of requestIds) - expect(locators.some((locator) => locator.endsWith(`@${id}`))).toBe(true) - } finally { - vi.unstubAllGlobals() + try { + if (failSecond) { + await expect(runDrain(drainId, 'cron')).rejects.toThrow(/HTTP 403/) + const checkpoint = (await drainRow()).cursor + expect(checkpoint).not.toBeNull() + expect(JSON.parse(checkpoint ?? '{}').id).toBe(acceptedIds.at(-1)) + } + await runDrain(drainId, 'cron') + expect(acceptedIds).toEqual( + Array.from({ length: 6 }, (_, index) => `${drainId}-${index.toString().padStart(6, '0')}`) + ) + const runs = await db.select().from(dataDrainRuns).where(eq(dataDrainRuns.drainId, drainId)) + const locators = runs.flatMap((run) => run.locators ?? []) + expect(new Set(locators).size).toBe(requestIds.length) + if (requestIdsReturned) { + for (const id of requestIds) + expect(locators.some((locator) => locator.endsWith(`@${id}`))).toBe(true) + } + } finally { + vi.unstubAllGlobals() + } } - }) + ) it('fails closed when malformed JSON credentials expose only a private-key fragment', async () => { await seedRows(1)