From 15b40aa17ef674f5760f5caf715d573281abcb35 Mon Sep 17 00:00:00 2001 From: Mariano Fuentes Date: Wed, 27 May 2026 13:45:36 -0400 Subject: [PATCH 1/2] fix(evidence-export): stream runs through PDF/JSON generation to prevent OOM The previous OOM fix loaded automations one at a time but still accumulated all runs for a single automation in memory. For orgs with large cloud security check histories, a single automation's runs could exceed the 6GB heap limit. Now uses async generators to stream run batches (50 at a time) through PDF and JSON generation. Peak memory is bounded by one batch of runs + the jsPDF document, regardless of total automation size. - evidence-data-loader: add streamAutomationRuns async generator - evidence-pdf-generator: extract renderRunToPDF, add generateAutomationPDFFromStream - evidence-json-builder: add buildAutomationJsonStream using Readable.from() - evidence-export.service: wire streaming into ZIP export path Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evidence-export/evidence-data-loader.ts | 55 ++-- .../evidence-export.service.spec.ts | 12 + .../evidence-export.service.ts | 82 +++-- .../evidence-export/evidence-json-builder.ts | 53 ++++ .../evidence-export/evidence-pdf-generator.ts | 293 +++++++++--------- 5 files changed, 298 insertions(+), 197 deletions(-) diff --git a/apps/api/src/tasks/evidence-export/evidence-data-loader.ts b/apps/api/src/tasks/evidence-export/evidence-data-loader.ts index 6d0da40c56..45fa235eac 100644 --- a/apps/api/src/tasks/evidence-export/evidence-data-loader.ts +++ b/apps/api/src/tasks/evidence-export/evidence-data-loader.ts @@ -197,22 +197,38 @@ export async function loadFullAutomation({ taskId: string; header: NormalizedAutomation; }): Promise { + const runs: NormalizedEvidenceRun[] = []; + for await (const batch of streamAutomationRuns({ taskId, header })) { + runs.push(...batch); + } + return { ...header, runs }; +} + +// Yields runs in batches so callers can process incrementally without +// accumulating all runs in memory. Each batch is GC-eligible after processing. +export async function* streamAutomationRuns({ + taskId, + header, +}: { + taskId: string; + header: NormalizedAutomation; +}): AsyncGenerator { if (header.type === 'app_automation' && header.checkId) { - return loadAppAutomationRuns(taskId, header); + yield* streamAppRuns(taskId, header.checkId); + } else { + yield* streamCustomRuns(taskId, header.id); } - return loadCustomAutomationRuns(taskId, header); } -async function loadAppAutomationRuns( +async function* streamAppRuns( taskId: string, - header: NormalizedAutomation, -): Promise { - const runs: NormalizedEvidenceRun[] = []; + checkId: string, +): AsyncGenerator { let cursor: { id: string } | undefined; for (;;) { const batch = await db.integrationCheckRun.findMany({ - where: { taskId, checkId: header.checkId }, + where: { taskId, checkId }, include: { results: true, connection: { include: { provider: true } }, @@ -224,28 +240,25 @@ async function loadAppAutomationRuns( if (batch.length === 0) break; - for (const run of batch) { - runs.push(normalizeAppAutomationRun(toAppAutomationRun(run))); - } + yield batch.map((run) => + normalizeAppAutomationRun(toAppAutomationRun(run)), + ); if (batch.length < RUN_BATCH_SIZE) break; cursor = { id: batch[batch.length - 1].id }; } - - return { ...header, runs }; } -async function loadCustomAutomationRuns( +async function* streamCustomRuns( taskId: string, - header: NormalizedAutomation, -): Promise { - const runs: NormalizedEvidenceRun[] = []; + automationId: string, +): AsyncGenerator { let cursor: { id: string } | undefined; for (;;) { const batch = await db.evidenceAutomationRun.findMany({ where: { - evidenceAutomation: { id: header.id, taskId }, + evidenceAutomation: { id: automationId, taskId }, version: { not: null }, }, include: { @@ -258,15 +271,13 @@ async function loadCustomAutomationRuns( if (batch.length === 0) break; - for (const run of batch) { - runs.push(normalizeCustomAutomationRun(toCustomAutomationRun(run))); - } + yield batch.map((run) => + normalizeCustomAutomationRun(toCustomAutomationRun(run)), + ); if (batch.length < RUN_BATCH_SIZE) break; cursor = { id: batch[batch.length - 1].id }; } - - return { ...header, runs }; } // Prisma result → normalizer interface mappers (single source of truth). diff --git a/apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts b/apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts index ab6de156d8..8b47882611 100644 --- a/apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts +++ b/apps/api/src/tasks/evidence-export/evidence-export.service.spec.ts @@ -87,6 +87,18 @@ jest.mock('@db', () => ({ jest.mock('./evidence-pdf-generator', () => ({ generateTaskSummaryPDF: jest.fn(() => Buffer.from('SUMMARY-PDF')), generateAutomationPDF: jest.fn(() => Buffer.from('AUTOMATION-PDF')), + generateAutomationPDFFromStream: jest.fn( + async ( + _header: unknown, + _context: unknown, + runBatches: AsyncIterable, + ) => { + for await (const _batch of runBatches) { + /* drain so underlying DB queries execute */ + } + return Buffer.from('AUTOMATION-PDF'); + }, + ), sanitizeFilename: (name: string) => name .toLowerCase() diff --git a/apps/api/src/tasks/evidence-export/evidence-export.service.ts b/apps/api/src/tasks/evidence-export/evidence-export.service.ts index cc77678d57..9781f06296 100644 --- a/apps/api/src/tasks/evidence-export/evidence-export.service.ts +++ b/apps/api/src/tasks/evidence-export/evidence-export.service.ts @@ -11,10 +11,11 @@ import type { } from './evidence-export.types'; import { generateAutomationPDF, + generateAutomationPDFFromStream, generateTaskSummaryPDF, sanitizeFilename, } from './evidence-pdf-generator'; -import { buildAutomationJson } from './evidence-json-builder'; +import { buildAutomationJson, buildAutomationJsonStream } from './evidence-json-builder'; import { appendAttachmentToArchive, createFilenameTracker, @@ -24,6 +25,7 @@ import { import { getAutomationHeaders, loadFullAutomation, + streamAutomationRuns, findTasksWithEvidence, } from './evidence-data-loader'; @@ -221,7 +223,8 @@ export class EvidenceExportService { }); } - // Loads each automation's runs individually so peak memory ≈ one automation, not all combined. + // Streams each automation's runs through PDF/JSON generation so peak memory + // is bounded by one batch of runs (~50) instead of the full automation. private async appendTaskContents(params: { archive: Archiver; headers: TaskEvidenceSummary; @@ -257,15 +260,10 @@ export class EvidenceExportService { } for (const automationHeader of headers.automations) { - const automation = await loadFullAutomation({ - taskId: headers.taskId, - header: automationHeader, - }); - - this.appendAutomationToArchive({ + await this.appendAutomationStreaming({ archive, headers, - automation, + automationHeader, folderName, options, perAutomationSubfolders, @@ -273,50 +271,70 @@ export class EvidenceExportService { } } - private appendAutomationToArchive(params: { + private async appendAutomationStreaming(params: { archive: Archiver; headers: TaskEvidenceSummary; - automation: NormalizedAutomation; + automationHeader: NormalizedAutomation; folderName: string; options: { includeRawJson?: boolean }; perAutomationSubfolders: boolean; - }): void { + }): Promise { const { archive, headers, - automation, + automationHeader, folderName, options, perAutomationSubfolders, } = params; const typePrefix = - automation.type === 'app_automation' ? 'app' : 'custom'; - const automationName = sanitizeFilename(automation.name); - const idSuffix = automation.id.slice(-8); - - const pdfBuffer = generateAutomationPDF(automation, { - organizationName: headers.organizationName, - taskTitle: headers.taskTitle, - }); - + automationHeader.type === 'app_automation' ? 'app' : 'custom'; + const automationName = sanitizeFilename(automationHeader.name); + const idSuffix = automationHeader.id.slice(-8); const basePath = perAutomationSubfolders ? `${folderName}/${typePrefix}-${automationName}-${idSuffix}` : folderName; - const pdfName = perAutomationSubfolders - ? `${basePath}/evidence.pdf` - : `${basePath}/${typePrefix}-${automationName}-${idSuffix}.pdf`; + const filePrefix = perAutomationSubfolders + ? `${basePath}/evidence` + : `${basePath}/${typePrefix}-${automationName}-${idSuffix}`; - archive.append(pdfBuffer, { name: pdfName }); + const context = { + organizationName: headers.organizationName, + taskTitle: headers.taskTitle, + }; if (options.includeRawJson) { - const jsonName = perAutomationSubfolders - ? `${basePath}/evidence.json` - : `${basePath}/${typePrefix}-${automationName}-${idSuffix}.json`; - archive.append( - Buffer.from(buildAutomationJson(headers, automation), 'utf-8'), - { name: jsonName }, + // Two independent DB cursors so neither PDF nor JSON buffers the full run set. + const pdfBuffer = await generateAutomationPDFFromStream( + automationHeader, + context, + streamAutomationRuns({ + taskId: headers.taskId, + header: automationHeader, + }), + ); + archive.append(pdfBuffer, { name: `${filePrefix}.pdf` }); + + const jsonStream = buildAutomationJsonStream({ + summary: headers, + header: automationHeader, + runBatches: streamAutomationRuns({ + taskId: headers.taskId, + header: automationHeader, + }), + }); + archive.append(jsonStream, { name: `${filePrefix}.json` }); + } else { + const pdfBuffer = await generateAutomationPDFFromStream( + automationHeader, + context, + streamAutomationRuns({ + taskId: headers.taskId, + header: automationHeader, + }), ); + archive.append(pdfBuffer, { name: `${filePrefix}.pdf` }); } } diff --git a/apps/api/src/tasks/evidence-export/evidence-json-builder.ts b/apps/api/src/tasks/evidence-export/evidence-json-builder.ts index 1f3c317d5f..e0d2914abf 100644 --- a/apps/api/src/tasks/evidence-export/evidence-json-builder.ts +++ b/apps/api/src/tasks/evidence-export/evidence-json-builder.ts @@ -1,8 +1,10 @@ +import { Readable } from 'node:stream'; import { configure as configureStringify } from 'safe-stable-stringify'; import { redactSensitiveData } from './evidence-redaction'; import type { TaskEvidenceSummary, NormalizedAutomation, + NormalizedEvidenceRun, } from './evidence-export.types'; const safeStringify = configureStringify({ @@ -11,6 +13,30 @@ const safeStringify = configureStringify({ deterministic: false, }); +function stringifyAutomationMeta(header: NormalizedAutomation): string { + return ( + safeStringify( + redactSensitiveData({ + id: header.id, + name: header.name, + type: header.type, + integrationName: header.integrationName, + totalRuns: header.totalRuns, + successfulRuns: header.successfulRuns, + failedRuns: header.failedRuns, + latestRunAt: header.latestRunAt, + }), + null, + 2, + ) ?? '{}' + ); +} + +function stringifyRun(run: NormalizedEvidenceRun): string { + const { type, automationName, automationId, ...rest } = run; + return safeStringify(redactSensitiveData(rest), null, 2) ?? '{}'; +} + export function buildAutomationJson( summary: TaskEvidenceSummary, automation: NormalizedAutomation, @@ -38,3 +64,30 @@ export function buildAutomationJson( ) ?? '{}' ); } + +export function buildAutomationJsonStream({ + summary, + header, + runBatches, +}: { + summary: TaskEvidenceSummary; + header: NormalizedAutomation; + runBatches: AsyncIterable; +}): Readable { + async function* chunks(): AsyncGenerator { + yield `{\n "automation": ${stringifyAutomationMeta(header)},\n "runs": [\n`; + + let first = true; + for await (const batch of runBatches) { + for (const run of batch) { + if (!first) yield ',\n'; + first = false; + yield ` ${stringifyRun(run)}`; + } + } + + yield `\n ],\n "exportedAt": ${JSON.stringify(summary.exportedAt.toISOString())}\n}\n`; + } + + return Readable.from(chunks(), { encoding: 'utf-8' }); +} diff --git a/apps/api/src/tasks/evidence-export/evidence-pdf-generator.ts b/apps/api/src/tasks/evidence-export/evidence-pdf-generator.ts index cde5bb2122..4196ed9e29 100644 --- a/apps/api/src/tasks/evidence-export/evidence-pdf-generator.ts +++ b/apps/api/src/tasks/evidence-export/evidence-pdf-generator.ts @@ -10,6 +10,7 @@ import stringify from 'safe-stable-stringify'; import { redactSensitiveData } from './evidence-redaction'; import type { NormalizedAutomation, + NormalizedEvidenceRun, TaskEvidenceSummary, } from './evidence-export.types'; @@ -233,18 +234,8 @@ function addPageNumbers(config: PDFConfig): void { } } -/** - * Generate PDF for a single automation - */ -export function generateAutomationPDF( - automation: NormalizedAutomation, - context: { - organizationName: string; - taskTitle: string; - }, -): Buffer { - const doc = new jsPDF(); - const config: PDFConfig = { +function createPDFConfig(doc: jsPDF): PDFConfig { + return { doc, pageWidth: doc.internal.pageSize.getWidth(), pageHeight: doc.internal.pageSize.getHeight(), @@ -254,25 +245,27 @@ export function generateAutomationPDF( defaultFontSize: 10, yPosition: 20, }; +} - // Header +function renderAutomationHeader( + config: PDFConfig, + header: NormalizedAutomation, + context: { organizationName: string; taskTitle: string }, +): void { addText(config, context.organizationName, { fontSize: 14, bold: true }); config.yPosition += config.lineHeight * 0.5; addText(config, `Task: ${context.taskTitle}`, { fontSize: 11 }); config.yPosition += config.lineHeight; - // Automation title const typeLabel = - automation.type === 'app_automation' - ? 'App Automation' - : 'Custom Automation'; - addText(config, `${typeLabel}: ${automation.name}`, { + header.type === 'app_automation' ? 'App Automation' : 'Custom Automation'; + addText(config, `${typeLabel}: ${header.name}`, { fontSize: 13, bold: true, }); - if (automation.integrationName) { - addText(config, `Integration: ${automation.integrationName}`, { + if (header.integrationName) { + addText(config, `Integration: ${header.integrationName}`, { fontSize: 10, color: [100, 100, 100], }); @@ -280,7 +273,6 @@ export function generateAutomationPDF( config.yPosition += config.lineHeight; - // Export timestamp addText(config, `Exported: ${format(new Date(), 'PPpp')}`, { fontSize: 9, color: [128, 128, 128], @@ -288,157 +280,172 @@ export function generateAutomationPDF( addSeparator(config); - // Summary section addSectionHeader(config, 'Summary'); - addText(config, `Total Runs: ${automation.totalRuns}`); - addText(config, `Successful: ${automation.successfulRuns}`); - addText(config, `Failed: ${automation.failedRuns}`); + addText(config, `Total Runs: ${header.totalRuns}`); + addText(config, `Successful: ${header.successfulRuns}`); + addText(config, `Failed: ${header.failedRuns}`); - if (automation.latestRunAt) { - addText(config, `Latest Run: ${format(automation.latestRunAt, 'PPpp')}`); + if (header.latestRunAt) { + addText(config, `Latest Run: ${format(header.latestRunAt, 'PPpp')}`); } addSeparator(config); - - // Runs section addSectionHeader(config, 'Run History'); +} - for (const run of automation.runs) { - checkPageBreak(config, config.lineHeight * 8); - - // Run header - const statusColor = getStatusColor(run.status, run.failedCount > 0); - addText(config, `Run: ${format(run.createdAt, 'PPpp')}`, { - fontSize: 10, - bold: true, - }); - addText(config, `Status: ${run.status.toUpperCase()}`, { - fontSize: 9, - color: statusColor, - }); +function renderRunToPDF(config: PDFConfig, run: NormalizedEvidenceRun): void { + checkPageBreak(config, config.lineHeight * 8); - if (run.durationMs) { - addText(config, `Duration: ${run.durationMs}ms`, { fontSize: 9 }); - } + const statusColor = getStatusColor(run.status, run.failedCount > 0); + addText(config, `Run: ${format(run.createdAt, 'PPpp')}`, { + fontSize: 10, + bold: true, + }); + addText(config, `Status: ${run.status.toUpperCase()}`, { + fontSize: 9, + color: statusColor, + }); - // Metrics - if (run.type === 'app_automation') { - addText( - config, - `Checked: ${run.totalChecked} | Passed: ${run.passedCount} | Failed: ${run.failedCount}`, - { fontSize: 9 }, - ); - } else if (run.evaluationStatus) { - addText(config, `Evaluation: ${run.evaluationStatus.toUpperCase()}`, { - fontSize: 9, - color: run.evaluationStatus === 'pass' ? [0, 128, 0] : [200, 0, 0], - }); - if (run.evaluationReason) { - addText(config, `Reason: ${run.evaluationReason}`, { - fontSize: 9, - indent: 10, - }); - } - } + if (run.durationMs) { + addText(config, `Duration: ${run.durationMs}ms`, { fontSize: 9 }); + } - // Error - if (run.error) { - config.yPosition += config.lineHeight * 0.5; - addText(config, 'Error:', { + if (run.type === 'app_automation') { + addText( + config, + `Checked: ${run.totalChecked} | Passed: ${run.passedCount} | Failed: ${run.failedCount}`, + { fontSize: 9 }, + ); + } else if (run.evaluationStatus) { + addText(config, `Evaluation: ${run.evaluationStatus.toUpperCase()}`, { + fontSize: 9, + color: run.evaluationStatus === 'pass' ? [0, 128, 0] : [200, 0, 0], + }); + if (run.evaluationReason) { + addText(config, `Reason: ${run.evaluationReason}`, { fontSize: 9, - bold: true, - color: [200, 0, 0], - }); - addText(config, run.error, { - fontSize: 8, - color: [150, 0, 0], indent: 10, }); } + } - // Output (for custom automations) - if (run.output) { - config.yPosition += config.lineHeight * 0.5; - addText(config, 'Output:', { fontSize: 9, bold: true }); - const outputText = formatJsonForPDF(run.output); - addMonospaceText(config, outputText); - } + if (run.error) { + config.yPosition += config.lineHeight * 0.5; + addText(config, 'Error:', { + fontSize: 9, + bold: true, + color: [200, 0, 0], + }); + addText(config, run.error, { + fontSize: 8, + color: [150, 0, 0], + indent: 10, + }); + } - // Logs - if (run.logs) { - config.yPosition += config.lineHeight * 0.5; - addText(config, 'Logs:', { fontSize: 9, bold: true }); - const logsText = formatJsonForPDF(run.logs); - addMonospaceText(config, logsText); - } + if (run.output) { + config.yPosition += config.lineHeight * 0.5; + addText(config, 'Output:', { fontSize: 9, bold: true }); + addMonospaceText(config, formatJsonForPDF(run.output)); + } - // Results (for app automations) - if (run.results.length > 0) { - config.yPosition += config.lineHeight * 0.5; - addText(config, `Results (${run.results.length}):`, { + if (run.logs) { + config.yPosition += config.lineHeight * 0.5; + addText(config, 'Logs:', { fontSize: 9, bold: true }); + addMonospaceText(config, formatJsonForPDF(run.logs)); + } + + if (run.results.length > 0) { + config.yPosition += config.lineHeight * 0.5; + addText(config, `Results (${run.results.length}):`, { + fontSize: 9, + bold: true, + }); + + for (const result of run.results) { + checkPageBreak(config, config.lineHeight * 5); + + const resultIcon = result.passed ? '[PASS]' : '[FAIL]'; + const resultColor: [number, number, number] = result.passed + ? [0, 100, 0] + : [180, 0, 0]; + + addText(config, `${resultIcon} ${result.title}`, { fontSize: 9, bold: true, + color: resultColor, + indent: 10, }); - for (const result of run.results) { - checkPageBreak(config, config.lineHeight * 5); - - const resultIcon = result.passed ? '[PASS]' : '[FAIL]'; - const resultColor: [number, number, number] = result.passed - ? [0, 100, 0] - : [180, 0, 0]; + addText( + config, + `Resource: ${result.resourceType}/${result.resourceId}`, + { fontSize: 8, indent: 15 }, + ); - addText(config, `${resultIcon} ${result.title}`, { - fontSize: 9, - bold: true, - color: resultColor, - indent: 10, + if (result.description) { + addText(config, result.description, { fontSize: 8, indent: 15 }); + } + if (result.severity) { + addText(config, `Severity: ${result.severity}`, { + fontSize: 8, + indent: 15, }); - - addText( - config, - `Resource: ${result.resourceType}/${result.resourceId}`, - { - fontSize: 8, - indent: 15, - }, - ); - - if (result.description) { - addText(config, result.description, { fontSize: 8, indent: 15 }); - } - - if (result.severity) { - addText(config, `Severity: ${result.severity}`, { - fontSize: 8, - indent: 15, - }); - } - - if (result.remediation) { - addText(config, `Remediation: ${result.remediation}`, { - fontSize: 8, - indent: 15, - }); - } - - if (result.evidence) { - addText(config, 'Evidence:', { fontSize: 8, bold: true, indent: 15 }); - const evidenceText = formatJsonForPDF(result.evidence); - addMonospaceText(config, evidenceText, 20); - } - - config.yPosition += config.lineHeight * 0.5; } - } + if (result.remediation) { + addText(config, `Remediation: ${result.remediation}`, { + fontSize: 8, + indent: 15, + }); + } + if (result.evidence) { + addText(config, 'Evidence:', { fontSize: 8, bold: true, indent: 15 }); + addMonospaceText(config, formatJsonForPDF(result.evidence), 20); + } - config.yPosition += config.lineHeight; - addSeparator(config); + config.yPosition += config.lineHeight * 0.5; + } } + config.yPosition += config.lineHeight; + addSeparator(config); +} + +/** + * Generate PDF for a single automation (loads all runs into memory). + * Use generateAutomationPDFFromStream for large automations. + */ +export function generateAutomationPDF( + automation: NormalizedAutomation, + context: { organizationName: string; taskTitle: string }, +): Buffer { + const config = createPDFConfig(new jsPDF()); + renderAutomationHeader(config, automation, context); + for (const run of automation.runs) { + renderRunToPDF(config, run); + } addPageNumbers(config); + return Buffer.from(config.doc.output('arraybuffer')); +} - return Buffer.from(doc.output('arraybuffer')); +/** + * Build a PDF incrementally from an async stream of run batches. + * Peak memory = one batch of runs + the jsPDF document (lightweight page objects). + */ +export async function generateAutomationPDFFromStream( + header: NormalizedAutomation, + context: { organizationName: string; taskTitle: string }, + runBatches: AsyncIterable, +): Promise { + const config = createPDFConfig(new jsPDF()); + renderAutomationHeader(config, header, context); + for await (const batch of runBatches) { + for (const run of batch) { + renderRunToPDF(config, run); + } + } + addPageNumbers(config); + return Buffer.from(config.doc.output('arraybuffer')); } /** From dd5a2f059d2d1121548d3c0fdd783550818bbab1 Mon Sep 17 00:00:00 2001 From: Mariano Fuentes Date: Thu, 28 May 2026 11:26:30 -0400 Subject: [PATCH 2/2] feat(evidence-export): offload bulk export to Trigger.dev background task The auditor bulk evidence export previously ran in the API process, peaking at ~20% memory per request. Multiple concurrent exports could OOM the container. Now the heavy work (DB queries, PDF generation, ZIP creation) runs in a Trigger.dev background task with its own memory. The API endpoint triggers the task and returns a runId for progress tracking. - Add export-organization-evidence Trigger.dev task (S3 upload + presigned URL) - Change POST /v1/evidence-export/all to trigger background task - Frontend uses useRealtimeRun for progress + auto-download on completion - API process memory stays flat regardless of export size Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evidence-export.controller.spec.ts | 52 ++-- .../evidence-export.controller.ts | 57 ++-- .../export-organization-evidence.ts | 275 ++++++++++++++++++ .../components/ExportEvidenceButton.tsx | 216 +++++++++++--- .../tasks/components/TasksPageClient.tsx | 54 +++- apps/app/src/lib/evidence-download.ts | 24 +- 6 files changed, 537 insertions(+), 141 deletions(-) create mode 100644 apps/api/src/trigger/evidence-export/export-organization-evidence.ts diff --git a/apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts b/apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts index e9f29c13e1..62015060f9 100644 --- a/apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts +++ b/apps/api/src/tasks/evidence-export/evidence-export.controller.spec.ts @@ -1,5 +1,10 @@ // Mocks must be declared before any SUT import so guards' transitive deps // (Prisma, better-auth) don't instantiate in Jest. +const mockTrigger = jest.fn(); +jest.mock('@trigger.dev/sdk', () => ({ + tasks: { trigger: mockTrigger }, +})); + jest.mock('@db', () => ({ ...jest.requireActual('@prisma/client'), db: {}, @@ -223,18 +228,15 @@ describe('EvidenceExportController', () => { describe('AuditorEvidenceExportController', () => { let controller: AuditorEvidenceExportController; - let service: jest.Mocked< - Pick - >; beforeEach(async () => { - service = { - streamOrganizationEvidenceZip: jest.fn(), - }; + mockTrigger.mockReset().mockResolvedValue({ + id: 'run_123', + publicAccessToken: 'tok_abc', + }); const moduleRef = await Test.createTestingModule({ controllers: [AuditorEvidenceExportController], - providers: [{ provide: EvidenceExportService, useValue: service }], }) .overrideGuard(HybridAuthGuard) .useValue({ canActivate: () => true }) @@ -245,34 +247,16 @@ describe('AuditorEvidenceExportController', () => { controller = moduleRef.get(AuditorEvidenceExportController); }); - it('pipes the org-wide archive to response with correct headers', async () => { - const archive = makeFakeArchive(); - service.streamOrganizationEvidenceZip.mockResolvedValue({ - archive: archive as unknown as import('archiver').Archiver, - filename: 'acme_all-evidence_2026-04-22.zip', - }); - const req = makeFakeRequest(); - const res = makeFakeResponse(); + it('triggers a background task and returns runId + token', async () => { + const result = await controller.exportAllEvidence('org_1', 'true'); - await controller.exportAllEvidence( - 'org_1', - 'true', - req as unknown as import('express').Request, - res as unknown as import('express').Response, - ); - - expect(service.streamOrganizationEvidenceZip).toHaveBeenCalledWith( - 'org_1', - { includeRawJson: true }, - ); - expect(res.setHeader).toHaveBeenCalledWith( - 'Content-Type', - 'application/zip', + expect(mockTrigger).toHaveBeenCalledWith( + 'export-organization-evidence', + { organizationId: 'org_1', includeJson: true }, ); - expect(res.setHeader).toHaveBeenCalledWith( - 'Content-Disposition', - `attachment; filename="acme_all-evidence_2026-04-22.zip"`, - ); - expect(archive.pipe).toHaveBeenCalledWith(res); + expect(result).toEqual({ + runId: 'run_123', + publicAccessToken: 'tok_abc', + }); }); }); diff --git a/apps/api/src/tasks/evidence-export/evidence-export.controller.ts b/apps/api/src/tasks/evidence-export/evidence-export.controller.ts index 3a5cd924d6..5eb3601dd3 100644 --- a/apps/api/src/tasks/evidence-export/evidence-export.controller.ts +++ b/apps/api/src/tasks/evidence-export/evidence-export.controller.ts @@ -1,6 +1,7 @@ import { Controller, Get, + Post, Param, Query, Req, @@ -16,6 +17,7 @@ import { ApiSecurity, ApiTags, } from '@nestjs/swagger'; +import { tasks } from '@trigger.dev/sdk'; import type { Request, Response } from 'express'; import type { Archiver } from 'archiver'; import { AuditRead } from '../../audit/skip-audit-log.decorator'; @@ -200,7 +202,8 @@ export class EvidenceExportController { } /** - * Auditor-only controller for bulk evidence export + * Auditor-only controller for bulk evidence export. + * The heavy work runs in a Trigger.dev background task to avoid OOM in the API. */ @ApiTags('Evidence Export (Auditor)') @Controller({ path: 'evidence-export', version: '1' }) @@ -209,18 +212,13 @@ export class EvidenceExportController { export class AuditorEvidenceExportController { private readonly logger = new Logger(AuditorEvidenceExportController.name); - constructor(private readonly evidenceExportService: EvidenceExportService) {} - - /** - * Export all evidence for the organization (auditor only) - */ - @Get('all') + @Post('all') @RequirePermission('evidence', 'read') @AuditRead() @ApiOperation({ - summary: 'Export all organization evidence as ZIP (Auditor only)', + summary: 'Trigger bulk evidence export (Auditor only)', description: - 'Generate and download a ZIP file containing all automation evidence across all tasks. Only accessible by auditors.', + 'Starts a background job that generates a ZIP of all evidence. Returns a run ID for progress tracking.', }) @ApiQuery({ name: 'includeJson', @@ -229,46 +227,27 @@ export class AuditorEvidenceExportController { required: false, }) @ApiResponse({ - status: 200, - description: 'ZIP file generated successfully', - content: { - 'application/zip': {}, - }, - }) - @ApiResponse({ - status: 403, - description: 'Access denied - Auditor role required', + status: 201, + description: 'Export job started', }) async exportAllEvidence( @OrganizationId() organizationId: string, @Query('includeJson') includeJson: string, - @Req() req: Request, - @Res() res: Response, ) { - this.logger.log('Auditor exporting all evidence', { + this.logger.log('Auditor triggering bulk evidence export', { organizationId, includeJson: includeJson === 'true', }); - const { archive, filename } = - await this.evidenceExportService.streamOrganizationEvidenceZip( - organizationId, - { includeRawJson: includeJson === 'true' }, - ); - - res.setHeader('Content-Type', 'application/zip'); - res.setHeader( - 'Content-Disposition', - `attachment; filename="${filename}"`, - ); - - pipeArchiveToResponse({ - archive, - req, - res, - logger: this.logger, - tag: `org ${organizationId}`, + const handle = await tasks.trigger('export-organization-evidence', { + organizationId, + includeJson: includeJson === 'true', }); + + return { + runId: handle.id, + publicAccessToken: handle.publicAccessToken, + }; } } diff --git a/apps/api/src/trigger/evidence-export/export-organization-evidence.ts b/apps/api/src/trigger/evidence-export/export-organization-evidence.ts new file mode 100644 index 0000000000..cefef6b1ca --- /dev/null +++ b/apps/api/src/trigger/evidence-export/export-organization-evidence.ts @@ -0,0 +1,275 @@ +import { metadata, schemaTask } from '@trigger.dev/sdk'; +import { z } from 'zod'; +import { + S3Client, + PutObjectCommand, + GetObjectCommand, +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import archiver from 'archiver'; +import { db } from '@db'; +import { format } from 'date-fns'; +import { + getAutomationHeaders, + streamAutomationRuns, + findTasksWithEvidence, +} from '@/tasks/evidence-export/evidence-data-loader'; +import { + generateAutomationPDFFromStream, + generateTaskSummaryPDF, + sanitizeFilename, +} from '@/tasks/evidence-export/evidence-pdf-generator'; +import { buildAutomationJsonStream } from '@/tasks/evidence-export/evidence-json-builder'; +import { + getTaskAttachments, + appendAttachmentToArchive, + createFilenameTracker, +} from '@/tasks/evidence-export/evidence-attachment-streamer'; +import { configure as configureStringify } from 'safe-stable-stringify'; + +const safeStringify = configureStringify({ + bigint: true, + circularValue: '[Circular]', + deterministic: false, +}); + +const PRESIGNED_URL_EXPIRY = 3600; + +function createS3Client(): S3Client { + const region = process.env.APP_AWS_REGION || 'us-east-1'; + const accessKeyId = process.env.APP_AWS_ACCESS_KEY_ID; + const secretAccessKey = process.env.APP_AWS_SECRET_ACCESS_KEY; + + if (!accessKeyId || !secretAccessKey) { + throw new Error( + 'AWS S3 credentials missing. Set APP_AWS_ACCESS_KEY_ID and APP_AWS_SECRET_ACCESS_KEY.', + ); + } + + return new S3Client({ + region, + credentials: { accessKeyId, secretAccessKey }, + ...(process.env.APP_AWS_ENDPOINT + ? { + endpoint: process.env.APP_AWS_ENDPOINT, + forcePathStyle: true, + } + : {}), + }); +} + +function getBucketName(): string { + const bucket = process.env.APP_AWS_BUCKET_NAME; + if (!bucket) throw new Error('APP_AWS_BUCKET_NAME is not set.'); + return bucket; +} + +export const exportOrganizationEvidenceTask = schemaTask({ + id: 'export-organization-evidence', + maxDuration: 60 * 30, + retry: { maxAttempts: 0 }, + schema: z.object({ + organizationId: z.string(), + includeJson: z.boolean().default(false), + }), + run: async ({ organizationId, includeJson }) => { + metadata.set('status', 'starting'); + metadata.set('progress', 0); + + const organization = await db.organization.findUnique({ + where: { id: organizationId }, + select: { name: true }, + }); + if (!organization) throw new Error('Organization not found'); + + const taskIds = await findTasksWithEvidence(organizationId); + if (taskIds.length === 0) throw new Error('No tasks with evidence found'); + + metadata.set('tasksTotal', taskIds.length); + metadata.set('tasksCompleted', 0); + metadata.set('status', 'generating'); + + const orgFolder = sanitizeFilename(organization.name); + const exportDate = format(new Date(), 'yyyy-MM-dd'); + const runId = metadata.get('runId') ?? crypto.randomUUID().slice(0, 8); + const s3Key = `${organizationId}/exports/evidence-${exportDate}-${runId}.zip`; + + const s3Client = createS3Client(); + const bucket = getBucketName(); + + const archive = archiver('zip', { zlib: { level: 6 } }); + const zipBuffer = await buildZipBuffer({ + archive, + organizationId, + organizationName: organization.name, + orgFolder, + taskIds, + includeJson, + }); + + metadata.set('status', 'uploading'); + + await s3Client.send( + new PutObjectCommand({ + Bucket: bucket, + Key: s3Key, + Body: zipBuffer, + ContentType: 'application/zip', + }), + ); + + metadata.set('status', 'generating-link'); + metadata.set('progress', 95); + + const downloadUrl = await getSignedUrl( + s3Client, + new GetObjectCommand({ Bucket: bucket, Key: s3Key }), + { expiresIn: PRESIGNED_URL_EXPIRY }, + ); + + metadata.set('status', 'completed'); + metadata.set('progress', 100); + metadata.set('downloadUrl', downloadUrl); + + return { downloadUrl, s3Key }; + }, +}); + +async function buildZipBuffer(params: { + archive: archiver.Archiver; + organizationId: string; + organizationName: string; + orgFolder: string; + taskIds: string[]; + includeJson: boolean; +}): Promise { + const chunks: Buffer[] = []; + params.archive.on('data', (chunk: Buffer) => chunks.push(chunk)); + const finished = new Promise((resolve, reject) => { + params.archive.on('end', resolve); + params.archive.on('error', reject); + }); + await populateArchive(params); + await finished; + return Buffer.concat(chunks); +} + +async function populateArchive({ + archive, + organizationId, + organizationName, + orgFolder, + taskIds, + includeJson, +}: { + archive: archiver.Archiver; + organizationId: string; + organizationName: string; + orgFolder: string; + taskIds: string[]; + includeJson: boolean; +}): Promise { + const manifestEntries: Array<{ + id: string; + title: string; + automations: number; + attachments: number; + }> = []; + let totalAttachments = 0; + + for (let i = 0; i < taskIds.length; i++) { + const taskId = taskIds[i]; + try { + const [headers, attachments] = await Promise.all([ + getAutomationHeaders({ organizationId, taskId }), + getTaskAttachments(organizationId, taskId), + ]); + + if (headers.automations.length === 0 && attachments.length === 0) { + continue; + } + + const taskIdSuffix = headers.taskId.slice(-8); + const taskFolder = `${orgFolder}/${sanitizeFilename(headers.taskTitle)}-${taskIdSuffix}`; + + const summaryPdf = generateTaskSummaryPDF(headers, { + attachmentsCount: attachments.length, + }); + archive.append(summaryPdf, { name: `${taskFolder}/00-summary.pdf` }); + + if (attachments.length > 0) { + const uniqueName = createFilenameTracker(); + for (const attachment of attachments) { + await appendAttachmentToArchive({ + archive, + attachment, + folderPath: `${taskFolder}/01-attachments`, + uniqueName, + }); + } + } + + for (const automationHeader of headers.automations) { + const typePrefix = + automationHeader.type === 'app_automation' ? 'app' : 'custom'; + const automationName = sanitizeFilename(automationHeader.name); + const idSuffix = automationHeader.id.slice(-8); + const filePrefix = `${taskFolder}/${typePrefix}-${automationName}-${idSuffix}`; + + const pdfBuffer = await generateAutomationPDFFromStream( + automationHeader, + { organizationName, taskTitle: headers.taskTitle }, + streamAutomationRuns({ taskId, header: automationHeader }), + ); + archive.append(pdfBuffer, { name: `${filePrefix}.pdf` }); + + if (includeJson) { + const jsonStream = buildAutomationJsonStream({ + summary: headers, + header: automationHeader, + runBatches: streamAutomationRuns({ + taskId, + header: automationHeader, + }), + }); + archive.append(jsonStream, { name: `${filePrefix}.json` }); + } + } + + manifestEntries.push({ + id: headers.taskId, + title: headers.taskTitle, + automations: headers.automations.length, + attachments: attachments.length, + }); + totalAttachments += attachments.length; + } catch (error) { + console.warn( + `Failed to export task ${taskId}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + metadata.set('tasksCompleted', i + 1); + metadata.set( + 'progress', + Math.round(((i + 1) / taskIds.length) * 90), + ); + } + + manifestEntries.sort((a, b) => a.title.localeCompare(b.title)); + + const manifest = { + organization: organizationName, + organizationId, + exportedAt: new Date().toISOString(), + tasksCount: manifestEntries.length, + totalAttachments, + tasks: manifestEntries, + }; + archive.append( + Buffer.from(safeStringify(manifest, null, 2) ?? '{}', 'utf-8'), + { name: `${orgFolder}/manifest.json` }, + ); + + await archive.finalize(); +} diff --git a/apps/app/src/app/(app)/[orgId]/auditor/(overview)/components/ExportEvidenceButton.tsx b/apps/app/src/app/(app)/[orgId]/auditor/(overview)/components/ExportEvidenceButton.tsx index 8d52fb693d..604d434031 100644 --- a/apps/app/src/app/(app)/[orgId]/auditor/(overview)/components/ExportEvidenceButton.tsx +++ b/apps/app/src/app/(app)/[orgId]/auditor/(overview)/components/ExportEvidenceButton.tsx @@ -1,6 +1,6 @@ 'use client'; -import { downloadAllEvidenceZip } from '@/lib/evidence-download'; +import { triggerBulkEvidenceExport } from '@/lib/evidence-download'; import { Button, HStack, @@ -14,78 +14,133 @@ import { Text, } from '@trycompai/design-system'; import { ArrowDown } from '@trycompai/design-system/icons'; -import { useState } from 'react'; +import { useRealtimeRun } from '@trigger.dev/react-hooks'; +import { useCallback, useState } from 'react'; import { toast } from 'sonner'; interface ExportEvidenceButtonProps { organizationName: string; } -export function ExportEvidenceButton({ organizationName }: ExportEvidenceButtonProps) { - const [isDownloading, setIsDownloading] = useState(false); +type ExportState = + | { phase: 'idle' } + | { phase: 'triggering' } + | { phase: 'running'; runId: string; accessToken: string }; + +export function ExportEvidenceButton({ + organizationName, +}: ExportEvidenceButtonProps) { const [includeJson, setIncludeJson] = useState(false); const [isOpen, setIsOpen] = useState(false); + const [exportState, setExportState] = useState({ + phase: 'idle', + }); + + const isRunning = + exportState.phase === 'triggering' || exportState.phase === 'running'; - const handleDownload = async () => { - setIsDownloading(true); + const handleTrigger = async () => { + setExportState({ phase: 'triggering' }); try { - await downloadAllEvidenceZip({ organizationName, includeJson }); - toast.success('Evidence package downloaded successfully'); - setIsOpen(false); + const { runId, publicAccessToken } = await triggerBulkEvidenceExport({ + includeJson, + }); + setExportState({ phase: 'running', runId, accessToken: publicAccessToken }); } catch (err) { - toast.error('Failed to download evidence. Please try again.'); - console.error('Evidence download error:', err); - } finally { - setIsDownloading(false); + toast.error('Failed to start evidence export. Please try again.'); + console.error('Evidence export trigger error:', err); + setExportState({ phase: 'idle' }); } }; + const handleComplete = useCallback( + (run: { output?: { downloadUrl?: string } | null; metadata?: Record }) => { + const downloadUrl = + run.output?.downloadUrl ?? + (run.metadata?.downloadUrl as string | undefined); + + if (downloadUrl) { + const link = document.createElement('a'); + link.href = downloadUrl; + link.download = `${organizationName || 'evidence'}-export.zip`; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + toast.success('Evidence package downloaded successfully'); + } else { + toast.error('Export completed but download link was not available.'); + } + + setExportState({ phase: 'idle' }); + setIsOpen(false); + }, + [organizationName], + ); + + const handleError = useCallback(() => { + toast.error('Evidence export failed. Please try again.'); + setExportState({ phase: 'idle' }); + }, []); + return ( <> - + { + if (!isRunning) setIsOpen(open); + }}> Export All Evidence - - Download every task's uploaded evidence as a single ZIP so - you can hand it to your auditor or keep an offline snapshot. - - - - - - Include raw JSON files - - - Adds machine-readable metadata alongside the evidence - files. + {exportState.phase === 'running' ? ( + + ) : ( + <> + + Download every task's uploaded evidence as a single ZIP + so you can hand it to your auditor or keep an offline + snapshot. - - - - - - - - + + + + + Include raw JSON files + + + Adds machine-readable metadata alongside the evidence + files. + + + + + + + + + + + )} @@ -93,3 +148,68 @@ export function ExportEvidenceButton({ organizationName }: ExportEvidenceButtonP ); } + +function ExportProgress({ + runId, + accessToken, + onComplete, + onError, +}: { + runId: string; + accessToken: string; + onComplete: (run: { output?: { downloadUrl?: string } | null; metadata?: Record }) => void; + onError: () => void; +}) { + const { run } = useRealtimeRun(runId, { + accessToken, + enabled: true, + onComplete: (run) => onComplete(run), + onError: () => onError(), + }); + + const meta = run?.metadata as + | { + status?: string; + progress?: number; + tasksCompleted?: number; + tasksTotal?: number; + } + | undefined; + + const progress = meta?.progress ?? 0; + const status = meta?.status ?? 'starting'; + const tasksCompleted = meta?.tasksCompleted ?? 0; + const tasksTotal = meta?.tasksTotal ?? 0; + + const statusLabel = + status === 'starting' + ? 'Starting export...' + : status === 'generating' + ? `Processing task ${tasksCompleted} of ${tasksTotal}...` + : status === 'generating-link' + ? 'Generating download link...' + : 'Preparing...'; + + return ( + + + {statusLabel} + +
+
+
+ + This may take a few minutes for large organizations. You can close this + dialog — the export will continue in the background. + + + ); +} diff --git a/apps/app/src/app/(app)/[orgId]/tasks/components/TasksPageClient.tsx b/apps/app/src/app/(app)/[orgId]/tasks/components/TasksPageClient.tsx index 4231a9e803..6550c92f88 100644 --- a/apps/app/src/app/(app)/[orgId]/tasks/components/TasksPageClient.tsx +++ b/apps/app/src/app/(app)/[orgId]/tasks/components/TasksPageClient.tsx @@ -1,7 +1,8 @@ 'use client'; import { UpdateOrganizationEvidenceApproval } from '@/components/forms/organization/update-organization-evidence-approval'; -import { downloadAllEvidenceZip } from '@/lib/evidence-download'; +import { triggerBulkEvidenceExport } from '@/lib/evidence-download'; +import { useRealtimeRun } from '@trigger.dev/react-hooks'; import type { Member, Task, User } from '@db'; import { Button, @@ -68,31 +69,64 @@ export function TasksPageClient({ const { tasks, createTask, mutate: mutateTasks } = useTasks({ initialData: initialTasks }); const { hasPermission } = usePermissions(); const [isCreateSheetOpen, setIsCreateSheetOpen] = useState(false); - const [isDownloadingAll, setIsDownloadingAll] = useState(false); const [includeRawJson, setIncludeRawJson] = useState(false); const [isPopoverOpen, setIsPopoverOpen] = useState(false); const [mainTab, setMainTab] = useState('evidence-list'); + const [exportRun, setExportRun] = useState<{ + runId: string; + accessToken: string; + } | null>(null); + const [isTriggering, setIsTriggering] = useState(false); + + const { run: realtimeRun } = useRealtimeRun(exportRun?.runId ?? '', { + accessToken: exportRun?.accessToken, + enabled: !!exportRun, + onComplete: (run) => { + const downloadUrl = + run.output?.downloadUrl ?? + (run.metadata?.downloadUrl as string | undefined); + if (downloadUrl) { + const link = document.createElement('a'); + link.href = downloadUrl; + link.download = `${organizationName || 'evidence'}-export.zip`; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + toast.success('Evidence package downloaded successfully'); + } + setExportRun(null); + setIsPopoverOpen(false); + }, + onError: () => { + toast.error('Evidence export failed. Please try again.'); + setExportRun(null); + }, + }); + + const isDownloadingAll = isTriggering || !!exportRun; const handleDownloadAllEvidence = async () => { - setIsDownloadingAll(true); + setIsTriggering(true); try { - await downloadAllEvidenceZip({ - organizationName: organizationName ?? undefined, + const result = await triggerBulkEvidenceExport({ includeJson: includeRawJson, }); - toast.success('Evidence package downloaded successfully'); - setIsPopoverOpen(false); + setExportRun({ + runId: result.runId, + accessToken: result.publicAccessToken, + }); + toast.info('Evidence export started. You\'ll be notified when it\'s ready.'); } catch (err) { const noEvidence = err instanceof Error && err.message?.includes('No tasks with evidence found'); if (noEvidence) { toast.info('No tasks with evidence found to export.'); } else { - toast.error('Failed to download evidence. Please try again.'); + toast.error('Failed to start evidence export. Please try again.'); } - console.error('Evidence download error:', err); + console.error('Evidence export error:', err); } finally { - setIsDownloadingAll(false); + setIsTriggering(false); } }; diff --git a/apps/app/src/lib/evidence-download.ts b/apps/app/src/lib/evidence-download.ts index 0081126dff..b11f229d24 100644 --- a/apps/app/src/lib/evidence-download.ts +++ b/apps/app/src/lib/evidence-download.ts @@ -45,24 +45,28 @@ export async function downloadTaskEvidenceZip({ } /** - * Download all evidence for the organization (auditor only) + * Trigger bulk evidence export as a background job. + * Returns a run ID and access token for tracking progress via Trigger.dev realtime. */ -export async function downloadAllEvidenceZip({ - organizationName, +export async function triggerBulkEvidenceExport({ includeJson = false, }: { - organizationName?: string; includeJson?: boolean; -}): Promise { +}): Promise<{ runId: string; publicAccessToken: string }> { const baseUrl = env.NEXT_PUBLIC_API_URL || 'http://localhost:3333'; const endpoint = `/v1/evidence-export/all?includeJson=${includeJson}`; - await downloadFile(baseUrl + endpoint, { - fallbackBaseName: organizationName - ? `${organizationName}-all-evidence` - : 'all-evidence', - fallbackExtension: 'zip', + const response = await fetch(baseUrl + endpoint, { + method: 'POST', + credentials: 'include', }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(error || `Failed to start export: ${response.statusText}`); + } + + return response.json(); } /**