diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index 302f81c3a6a..42e11b0cd34 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -1,18 +1,14 @@ import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' -import { sha256Hex } from '@sim/security/hash' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' import { fileServeParamsSchema, fileServeQuerySchema } from '@/lib/api/contracts/storage-transfer' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { DocCompileUserError, - getE2BDocFormat, - loadCompiledDocByExt, + resolveServableDocBytes, } from '@/lib/copilot/tools/server/files/doc-compile' -import { isE2BDocEnabled } from '@/lib/core/config/env-flags' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { runSandboxTask } from '@/lib/execution/sandbox/run-task' import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads' import type { StorageContext } from '@/lib/uploads/config' import { parseWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager' @@ -26,47 +22,14 @@ import { findLocalFile, getContentType, } from '@/app/api/files/utils' -import type { SandboxTaskId } from '@/sandbox-tasks/registry' const logger = createLogger('FilesServeAPI') -const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04]) -const PDF_MAGIC = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d]) // %PDF- - -interface CompilableFormat { - magic: Buffer - taskId: SandboxTaskId - contentType: string -} - -const COMPILABLE_FORMATS: Record = { - '.pptx': { - magic: ZIP_MAGIC, - taskId: 'pptx-generate', - contentType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - }, - '.docx': { - magic: ZIP_MAGIC, - taskId: 'docx-generate', - contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - }, - '.pdf': { - magic: PDF_MAGIC, - taskId: 'pdf-generate', - contentType: 'application/pdf', - }, -} - -const MAX_COMPILED_DOC_CACHE = 10 -const compiledDocCache = new Map() - -function compiledCacheSet(key: string, buffer: Buffer): void { - if (compiledDocCache.size >= MAX_COMPILED_DOC_CACHE) { - compiledDocCache.delete(compiledDocCache.keys().next().value as string) - } - compiledDocCache.set(key, buffer) -} - +/** + * Resolves the bytes + content type to serve for a stored file via the shared + * {@link resolveServableDocBytes} (generated docs → compiled artifact). `raw=1` + * bypasses resolution and serves the stored source as-is. + */ async function compileDocumentIfNeeded( buffer: Buffer, filename: string, @@ -76,71 +39,13 @@ async function compileDocumentIfNeeded( signal: AbortSignal | undefined ): Promise<{ buffer: Buffer; contentType: string }> { if (raw) return { buffer, contentType: getContentType(filename) } - - const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase() - const extNoDot = ext.replace(/^\./, '') - const format = COMPILABLE_FORMATS[ext] - - // Already a binary file (uploaded or pre-compiled)? Serve as-is. - if (format) { - const magicLen = format.magic.length - if (buffer.length >= magicLen && buffer.subarray(0, magicLen).equals(format.magic)) { - return { buffer, contentType: getContentType(filename) } - } - } - - // .xlsx is a ZIP container with no JS compile path. An uploaded/binary xlsx - // must short-circuit here (it isn't in COMPILABLE_FORMATS) — otherwise every - // xlsx open would utf-8-decode the whole binary and do an always-miss S3 GET. - // Only a Python-source xlsx (UTF-8 text, no ZIP magic) falls through. - if ( - extNoDot === 'xlsx' && - buffer.length >= ZIP_MAGIC.length && - buffer.subarray(0, ZIP_MAGIC.length).equals(ZIP_MAGIC) - ) { - return { buffer, contentType: getContentType(filename) } - } - - // Generated docs render from a content-addressed compiled binary that is built - // exactly ONCE per edit_content/create (at write time) and stored in S3. Serve - // only LOADS it — it must never compile, or it would re-run E2B on every preview - // fetch, including against the incomplete source mid-generation. A hit returns - // the (possibly partial) committed doc; a miss in the E2B regime means the doc - // is still being generated → 409, and the client polls until the artifact lands. - if (workspaceId && (format || extNoDot === 'xlsx')) { - const source = buffer.toString('utf-8') - // Load the prebuilt artifact directly from S3 (content-addressed). No extra - // in-memory layer here: the store is the source of truth, the client (react - // query) already caches the bytes, and this branch never recomputes. - const stored = await loadCompiledDocByExt(workspaceId, source, extNoDot) - if (stored) { - return { buffer: stored.buffer, contentType: stored.contentType } - } - - if (isE2BDocEnabled && (await getE2BDocFormat(filename))) { - // Artifact not built yet (still generating, or the source didn't compile at - // write time). Signal "not ready" without compiling — handled as 409. - throw new DocCompileUserError('Document is still being generated') - } - } - - if (!format) return { buffer, contentType: getContentType(filename) } - - // E2B disabled and no stored artifact → compile JS source via isolated-vm. - const code = buffer.toString('utf-8') - const cacheKey = sha256Hex(`${ext}${code}${workspaceId ?? ''}`) - const cached = compiledDocCache.get(cacheKey) - if (cached) { - return { buffer: cached, contentType: format.contentType } - } - - const compiled = await runSandboxTask( - format.taskId, - { code, workspaceId: workspaceId || '' }, - { ownerKey, signal } - ) - compiledCacheSet(cacheKey, compiled) - return { buffer: compiled, contentType: format.contentType } + return resolveServableDocBytes({ + rawBuffer: buffer, + fileName: filename, + workspaceId, + ownerKey, + signal, + }) } const STORAGE_KEY_PREFIX_RE = /^\d{13}-[a-z0-9]{7}-/ diff --git a/apps/sim/app/api/tools/a2a/send-message/route.ts b/apps/sim/app/api/tools/a2a/send-message/route.ts index 6c73c791c3e..4c098b98fda 100644 --- a/apps/sim/app/api/tools/a2a/send-message/route.ts +++ b/apps/sim/app/api/tools/a2a/send-message/route.ts @@ -17,7 +17,8 @@ import { enforceUserOrIpRateLimit } from '@/lib/core/rate-limiter' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -90,13 +91,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied } files = await Promise.all( - userFiles.map(async (userFile) => ({ - bytes: await downloadFileFromStorage(userFile, requestId, logger, { - maxBytes: A2A_MAX_FILE_BYTES, - }), - name: userFile.name, - mediaType: userFile.type || 'application/octet-stream', - })) + userFiles.map(async (userFile) => { + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger, + { maxBytes: A2A_MAX_FILE_BYTES } + ) + return { + bytes: buffer, + name: userFile.name, + mediaType: contentType || userFile.type || 'application/octet-stream', + } + }) ) } @@ -130,6 +137,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { output, }) } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] A2A send failed`, { error: getErrorMessage(error) }) return NextResponse.json({ success: false, error: getErrorMessage(error) }, { status: 502 }) } diff --git a/apps/sim/app/api/tools/agiloft/attach/route.test.ts b/apps/sim/app/api/tools/agiloft/attach/route.test.ts index f1e4c8c4264..3f1ac2d1fae 100644 --- a/apps/sim/app/api/tools/agiloft/attach/route.test.ts +++ b/apps/sim/app/api/tools/agiloft/attach/route.test.ts @@ -21,7 +21,7 @@ vi.mock('@/lib/uploads/utils/file-utils', () => ({ processFilesToUserFiles: mockProcessFilesToUserFiles, })) vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadFileFromStorage: mockDownloadFileFromStorage, + downloadServableFileFromStorage: mockDownloadFileFromStorage, })) vi.mock('@/app/api/files/authorization', () => ({ assertToolFileAccess: mockAssertToolFileAccess, @@ -77,7 +77,10 @@ beforeEach(() => { { key: 's3://bucket/file.txt', name: 'file.txt', size: 5, type: 'text/plain' }, ]) mockAssertToolFileAccess.mockResolvedValue(null) - mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('hello')) + mockDownloadFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('hello'), + contentType: 'application/octet-stream', + }) }) describe('POST /api/tools/agiloft/attach', () => { diff --git a/apps/sim/app/api/tools/agiloft/attach/route.ts b/apps/sim/app/api/tools/agiloft/attach/route.ts index b0fcb351751..64da0e6acbb 100644 --- a/apps/sim/app/api/tools/agiloft/attach/route.ts +++ b/apps/sim/app/api/tools/agiloft/attach/route.ts @@ -9,7 +9,8 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { buildAttachFileUrl } from '@/tools/agiloft/utils' import { @@ -74,7 +75,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + + let fileBuffer: Buffer + try { + const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = servable.buffer + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download file from storage:`, error) + return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) + } + const resolvedFileName = data.fileName || userFile.name || 'attachment' let resolvedIP: string diff --git a/apps/sim/app/api/tools/box/upload/route.ts b/apps/sim/app/api/tools/box/upload/route.ts index 95ca9979054..57ec5f4223c 100644 --- a/apps/sim/app/api/tools/box/upload/route.ts +++ b/apps/sim/app/api/tools/box/upload/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -53,7 +54,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + try { + const result = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = result.buffer + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to download file') }, + { status: 500 } + ) + } fileName = validatedData.fileName || userFile.name } else if (validatedData.fileContent) { logger.info(`[${requestId}] Using legacy base64 content input`) diff --git a/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts b/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts index 4453ddc1dfa..eae6759b6cb 100644 --- a/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts +++ b/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts @@ -21,7 +21,7 @@ vi.mock('@/lib/uploads/utils/file-utils', () => ({ processFilesToUserFiles: mockProcessFilesToUserFiles, })) vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadFileFromStorage: mockDownloadFileFromStorage, + downloadServableFileFromStorage: mockDownloadFileFromStorage, })) vi.mock('@/app/api/files/authorization', () => ({ assertToolFileAccess: mockAssertToolFileAccess, @@ -65,7 +65,10 @@ beforeEach(() => { { key: 'uploads/receipt.pdf', name: 'receipt.pdf', size: 5, type: 'application/pdf' }, ]) mockAssertToolFileAccess.mockResolvedValue(null) - mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('receipt-bytes')) + mockDownloadFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('receipt-bytes'), + contentType: 'application/pdf', + }) }) describe('POST /api/tools/brex/upload-receipt', () => { @@ -192,7 +195,10 @@ describe('POST /api/tools/brex/upload-receipt', () => { }) it('rejects files over the 50 MB limit', async () => { - mockDownloadFileFromStorage.mockResolvedValueOnce(Buffer.alloc(50 * 1024 * 1024 + 1)) + mockDownloadFileFromStorage.mockResolvedValueOnce({ + buffer: Buffer.alloc(50 * 1024 * 1024 + 1), + contentType: 'application/pdf', + }) const response = await POST(createMockRequest('POST', baseBody)) expect(response.status).toBe(400) diff --git a/apps/sim/app/api/tools/brex/upload-receipt/route.ts b/apps/sim/app/api/tools/brex/upload-receipt/route.ts index 792e9ab6d52..6fb4ccfecf8 100644 --- a/apps/sim/app/api/tools/brex/upload-receipt/route.ts +++ b/apps/sim/app/api/tools/brex/upload-receipt/route.ts @@ -11,7 +11,8 @@ import { import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { BREX_API_BASE, buildBrexHeaders } from '@/tools/brex/utils' @@ -48,7 +49,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + let fileBuffer: Buffer + try { + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = resolved.buffer + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download receipt file:`, error) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } if (fileBuffer.length > MAX_RECEIPT_SIZE_BYTES) { return NextResponse.json( { success: false, error: 'Receipt file exceeds the 50 MB limit' }, diff --git a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts index be4932a9964..00057e4d10c 100644 --- a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts +++ b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts @@ -7,7 +7,8 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processSingleFileToUserFile, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { getConfluenceCloudId } from '@/tools/confluence/utils' import { parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -91,9 +92,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied let fileBuffer: Buffer + let resolvedContentType: string try { - fileBuffer = await downloadFileFromStorage(userFile, 'confluence-upload', logger) + const servable = await downloadServableFileFromStorage(userFile, 'confluence-upload', logger) + fileBuffer = servable.buffer + resolvedContentType = servable.contentType } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error('Failed to download file from storage:', error) return NextResponse.json( { @@ -104,7 +110,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const uploadFileName = fileName || userFile.name || 'attachment' - const mimeType = userFile.type || 'application/octet-stream' + const mimeType = resolvedContentType || userFile.type || 'application/octet-stream' const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/${pageId}/child/attachment` diff --git a/apps/sim/app/api/tools/daytona/upload/route.ts b/apps/sim/app/api/tools/daytona/upload/route.ts index 52fb2ec70ad..5c52509bb75 100644 --- a/apps/sim/app/api/tools/daytona/upload/route.ts +++ b/apps/sim/app/api/tools/daytona/upload/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { DAYTONA_TOOLBOX_BASE_URL, extractDaytonaError } from '@/tools/daytona/utils' @@ -60,7 +61,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`) - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + try { + const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = servable.buffer + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download file from storage:`, error) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to download file') }, + { status: 500 } + ) + } fileName = params.fileName || userFile.name } else if (params.fileContent) { logger.info(`[${requestId}] Using legacy base64 content input`) diff --git a/apps/sim/app/api/tools/discord/send-message/route.ts b/apps/sim/app/api/tools/discord/send-message/route.ts index 5efc1b44d21..aa644457654 100644 --- a/apps/sim/app/api/tools/discord/send-message/route.ts +++ b/apps/sim/app/api/tools/discord/send-message/route.ts @@ -8,7 +8,8 @@ import { validateNumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -143,31 +144,39 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = accessResults.find((r) => r !== null) if (denied) return denied - const buffers = await Promise.all( - userFiles.map(async (file, i) => { - try { + let resolved: Array<{ buffer: Buffer; contentType: string }> + try { + resolved = await Promise.all( + userFiles.map(async (file, i) => { logger.info(`[${requestId}] Downloading file ${i}: ${file.name}`) - return await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - }) - ) + return await downloadServableFileFromStorage(file, requestId, logger) + }) + ) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download an attachment:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } for (let i = 0; i < userFiles.length; i++) { const userFile = userFiles[i] - const buffer = buffers[i] + const buffer = resolved[i].buffer + const mimeType = resolved[i].contentType || userFile.type || 'application/octet-stream' logger.info(`[${requestId}] Added file ${i}: ${userFile.name} (${buffer.length} bytes)`) filesOutput.push({ name: userFile.name, - mimeType: userFile.type || 'application/octet-stream', + mimeType, data: buffer.toString('base64'), size: buffer.length, }) - const blob = new Blob([new Uint8Array(buffer)], { type: userFile.type }) + const blob = new Blob([new Uint8Array(buffer)], { type: mimeType }) formData.append(`files[${i}]`, blob, userFile.name) } diff --git a/apps/sim/app/api/tools/docusign/route.ts b/apps/sim/app/api/tools/docusign/route.ts index 594587fc510..e36b6cee009 100644 --- a/apps/sim/app/api/tools/docusign/route.ts +++ b/apps/sim/app/api/tools/docusign/route.ts @@ -17,7 +17,8 @@ import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' const logger = createLogger('DocuSignAPI') @@ -228,14 +229,21 @@ async function handleSendEnvelope( { status: 413 } ) } - const buffer = await downloadFileFromStorage(userFile, 'docusign-send', logger, { - maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES, - }) + const { buffer } = await downloadServableFileFromStorage( + userFile, + 'docusign-send', + logger, + { + maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES, + } + ) assertKnownSizeWithinLimit(buffer.length, MAX_DOCUSIGN_DOCUMENT_BYTES, 'DocuSign document') documentBase64 = buffer.toString('base64') documentName = userFile.name } } catch (fileError) { + const notReady = docNotReadyResponse(fileError) + if (notReady) return notReady logger.error('Failed to process file for DocuSign envelope', { fileError }) return NextResponse.json( { diff --git a/apps/sim/app/api/tools/dropbox/upload/route.ts b/apps/sim/app/api/tools/dropbox/upload/route.ts index d9f375c819a..c68967b839b 100644 --- a/apps/sim/app/api/tools/dropbox/upload/route.ts +++ b/apps/sim/app/api/tools/dropbox/upload/route.ts @@ -8,7 +8,8 @@ import { generateRequestId } from '@/lib/core/utils/request' import { httpHeaderSafeJson } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -56,7 +57,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + try { + const result = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = result.buffer + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to download file') }, + { status: 500 } + ) + } fileName = userFile.name } else if (validatedData.fileContent) { // Legacy: base64 string input (backwards compatibility) diff --git a/apps/sim/app/api/tools/file/manage/route.ts b/apps/sim/app/api/tools/file/manage/route.ts index 367a5db8cfc..4a2b9e260b1 100644 --- a/apps/sim/app/api/tools/file/manage/route.ts +++ b/apps/sim/app/api/tools/file/manage/route.ts @@ -31,7 +31,11 @@ import { } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { + downloadFileFromStorage, + downloadServableFileFromStorage, +} from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { performMoveWorkspaceFileItems } from '@/lib/workspace-files/orchestration' import { assertActiveWorkspaceAccess, @@ -306,7 +310,7 @@ const extractUserFileTextContent = async ( userFile: UserFile, requestId: string ): Promise => { - const buffer = await downloadFileFromStorage(userFile, requestId, logger, { + const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { maxBytes: MAX_GET_CONTENT_FILE_BYTES, }) @@ -1038,6 +1042,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 403 } ) } + const notReady = docNotReadyResponse(error) + if (notReady) return notReady if (error instanceof ShareValidationError) { return NextResponse.json({ success: false, error: error.message }, { status: 400 }) } diff --git a/apps/sim/app/api/tools/firecrawl/parse/route.ts b/apps/sim/app/api/tools/firecrawl/parse/route.ts index 409f74a6f16..610df2f45a2 100644 --- a/apps/sim/app/api/tools/firecrawl/parse/route.ts +++ b/apps/sim/app/api/tools/firecrawl/parse/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -47,11 +48,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger + ) const formData = new FormData() const blob = new Blob([new Uint8Array(buffer)], { - type: userFile.type || 'application/octet-stream', + type: contentType || userFile.type || 'application/octet-stream', }) formData.append('file', blob, userFile.name) @@ -88,6 +93,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { output: firecrawlData.data ?? firecrawlData, }) } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Error in Firecrawl parse:`, error) return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) } diff --git a/apps/sim/app/api/tools/gmail/draft/route.ts b/apps/sim/app/api/tools/gmail/draft/route.ts index ca4ab7b2599..e376be1f01e 100644 --- a/apps/sim/app/api/tools/gmail/draft/route.ts +++ b/apps/sim/app/api/tools/gmail/draft/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { base64UrlEncode, @@ -94,26 +95,45 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = accessResults.find((r) => r !== null) if (denied) return denied - const buffers = await Promise.all( - attachments.map(async (file) => { - try { + let resolved: Array<{ buffer: Buffer; contentType: string }> + try { + resolved = await Promise.all( + attachments.map(async (file) => { logger.info( `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` ) - return await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - }) - ) + return await downloadServableFileFromStorage(file, requestId, logger) + }) + ) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download an attachment:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + + const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) + if (resolvedTotal > maxSize) { + const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, - mimeType: file.type || 'application/octet-stream', - content: buffers[i], + mimeType: resolved[i].contentType || file.type || 'application/octet-stream', + content: resolved[i].buffer, })) const mimeMessage = buildMimeMessage({ diff --git a/apps/sim/app/api/tools/gmail/edit-draft/route.ts b/apps/sim/app/api/tools/gmail/edit-draft/route.ts index cac63c2ab92..f986a610f11 100644 --- a/apps/sim/app/api/tools/gmail/edit-draft/route.ts +++ b/apps/sim/app/api/tools/gmail/edit-draft/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { base64UrlEncode, @@ -90,26 +91,45 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = accessResults.find((r) => r !== null) if (denied) return denied - const buffers = await Promise.all( - attachments.map(async (file) => { - try { + let resolved: Array<{ buffer: Buffer; contentType: string }> + try { + resolved = await Promise.all( + attachments.map(async (file) => { logger.info( `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` ) - return await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - }) - ) + return await downloadServableFileFromStorage(file, requestId, logger) + }) + ) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download an attachment:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + + const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) + if (resolvedTotal > maxSize) { + const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, - mimeType: file.type || 'application/octet-stream', - content: buffers[i], + mimeType: resolved[i].contentType || file.type || 'application/octet-stream', + content: resolved[i].buffer, })) const mimeMessage = buildMimeMessage({ diff --git a/apps/sim/app/api/tools/gmail/send/route.ts b/apps/sim/app/api/tools/gmail/send/route.ts index 81818bfc98e..59df7377b34 100644 --- a/apps/sim/app/api/tools/gmail/send/route.ts +++ b/apps/sim/app/api/tools/gmail/send/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { base64UrlEncode, @@ -94,26 +95,45 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = accessResults.find((r) => r !== null) if (denied) return denied - const buffers = await Promise.all( - attachments.map(async (file) => { - try { + let resolved: Array<{ buffer: Buffer; contentType: string }> + try { + resolved = await Promise.all( + attachments.map(async (file) => { logger.info( `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` ) - return await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - }) - ) + return await downloadServableFileFromStorage(file, requestId, logger) + }) + ) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download an attachment:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + + const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) + if (resolvedTotal > maxSize) { + const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + }, + { status: 400 } + ) + } const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, - mimeType: file.type || 'application/octet-stream', - content: buffers[i], + mimeType: resolved[i].contentType || file.type || 'application/octet-stream', + content: resolved[i].buffer, })) const mimeMessage = buildMimeMessage({ diff --git a/apps/sim/app/api/tools/google_drive/upload/route.ts b/apps/sim/app/api/tools/google_drive/upload/route.ts index 9c0cd1ccd9f..0600386324e 100644 --- a/apps/sim/app/api/tools/google_drive/upload/route.ts +++ b/apps/sim/app/api/tools/google_drive/upload/route.ts @@ -8,7 +8,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { GOOGLE_WORKSPACE_MIME_TYPES, @@ -120,10 +121,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied let fileBuffer: Buffer + let downloadedContentType = '' try { - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = result.buffer + downloadedContentType = result.contentType } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Failed to download file:`, error) return NextResponse.json( { @@ -134,8 +140,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - let uploadMimeType = validatedData.mimeType || userFile.type || 'application/octet-stream' - const requestedMimeType = validatedData.mimeType || userFile.type || 'application/octet-stream' + let uploadMimeType = + validatedData.mimeType || downloadedContentType || userFile.type || 'application/octet-stream' + const requestedMimeType = + validatedData.mimeType || downloadedContentType || userFile.type || 'application/octet-stream' if (GOOGLE_WORKSPACE_MIME_TYPES.includes(requestedMimeType)) { uploadMimeType = SOURCE_MIME_TYPES[requestedMimeType] || 'text/plain' diff --git a/apps/sim/app/api/tools/jira/add-attachment/route.ts b/apps/sim/app/api/tools/jira/add-attachment/route.ts index 2fc2e89f2a7..b23abf6656b 100644 --- a/apps/sim/app/api/tools/jira/add-attachment/route.ts +++ b/apps/sim/app/api/tools/jira/add-attachment/route.ts @@ -6,7 +6,8 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' @@ -47,9 +48,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { for (const file of userFiles) { const denied = await assertToolFileAccess(file.key, authResult.userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(file, requestId, logger) + let buffer: Buffer + let downloadedContentType = '' + try { + const result = await downloadServableFileFromStorage(file, requestId, logger) + buffer = result.buffer + downloadedContentType = result.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + throw error + } const blob = new Blob([new Uint8Array(buffer)], { - type: file.type || 'application/octet-stream', + type: downloadedContentType || file.type || 'application/octet-stream', }) formData.append('file', blob, file.name) } diff --git a/apps/sim/app/api/tools/linq/upload/route.ts b/apps/sim/app/api/tools/linq/upload/route.ts index 379b7665c0b..6b0004df4e2 100644 --- a/apps/sim/app/api/tools/linq/upload/route.ts +++ b/apps/sim/app/api/tools/linq/upload/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { extractLinqError, LINQ_API_BASE, linqHeaders } from '@/tools/linq/utils' @@ -58,9 +59,24 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const userFile = userFiles[0] const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - buffer = await downloadFileFromStorage(userFile, requestId, logger) + let resolvedContentTypeFromStorage: string + try { + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + buffer = resolved.buffer + resolvedContentTypeFromStorage = resolved.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download Linq attachment file:`, error) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status: 500 } + ) + } if (!resolvedFilename) resolvedFilename = userFile.name - if (!resolvedContentType) resolvedContentType = userFile.type || 'application/octet-stream' + if (!resolvedContentType) + resolvedContentType = + resolvedContentTypeFromStorage || userFile.type || 'application/octet-stream' } else if (fileContent) { buffer = Buffer.from(fileContent, 'base64') if (!resolvedFilename) resolvedFilename = 'file' diff --git a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts index dbf5b10093b..21a767f4a3e 100644 --- a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts +++ b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts @@ -8,7 +8,8 @@ import { secureFetchWithValidation } from '@/lib/core/security/input-validation. import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -71,7 +72,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + try { + const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = servable.buffer + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download file from storage:`, error) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to download file') }, + { status: 500 } + ) + } } else if (validatedData.fileContent) { fileBuffer = Buffer.from(validatedData.fileContent, 'base64') } else { diff --git a/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts b/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts index 58fb3cfd94a..cdce8f5f38e 100644 --- a/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts +++ b/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts @@ -7,6 +7,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { FileAccessDeniedError } from '@/app/api/files/authorization' import { uploadFilesForTeamsMessage } from '@/tools/microsoft_teams/server-utils' import type { GraphApiErrorResponse, GraphChatMessage } from '@/tools/microsoft_teams/types' @@ -167,6 +168,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (error instanceof FileAccessDeniedError) { return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) } + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Error sending Teams channel message:`, error) return NextResponse.json( { diff --git a/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts b/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts index 96a4dada98c..b4af6be45df 100644 --- a/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts +++ b/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts @@ -7,6 +7,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { FileAccessDeniedError } from '@/app/api/files/authorization' import { uploadFilesForTeamsMessage } from '@/tools/microsoft_teams/server-utils' import type { GraphApiErrorResponse, GraphChatMessage } from '@/tools/microsoft_teams/types' @@ -164,6 +165,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (error instanceof FileAccessDeniedError) { return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) } + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Error sending Teams chat message:`, error) return NextResponse.json( { diff --git a/apps/sim/app/api/tools/mistral/parse/route.ts b/apps/sim/app/api/tools/mistral/parse/route.ts index acfd066879a..f9ec00f344e 100644 --- a/apps/sim/app/api/tools/mistral/parse/route.ts +++ b/apps/sim/app/api/tools/mistral/parse/route.ts @@ -12,9 +12,10 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { - downloadFileFromStorage, + downloadServableFileFromStorage, resolveInternalFileUrl, } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -124,8 +125,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (!base64) { const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger + ) base64 = buffer.toString('base64') + if (contentType && contentType !== 'application/octet-stream') { + mimeType = contentType + } } const base64Payload = base64.startsWith('data:') ? base64 @@ -265,6 +273,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { output: mistralData, }) } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Error in Mistral parse:`, error) return NextResponse.json( diff --git a/apps/sim/app/api/tools/onedrive/upload/route.ts b/apps/sim/app/api/tools/onedrive/upload/route.ts index f27cc9aa4bc..2c94986de9f 100644 --- a/apps/sim/app/api/tools/onedrive/upload/route.ts +++ b/apps/sim/app/api/tools/onedrive/upload/route.ts @@ -13,7 +13,8 @@ import { getExtensionFromMimeType, processSingleFileToUserFile, } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { normalizeExcelValues } from '@/tools/onedrive/utils' @@ -114,8 +115,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied try { - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + const result = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = result.buffer + mimeType = result.contentType || userFile.type || 'application/octet-stream' } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Failed to download file from storage:`, error) return NextResponse.json( { @@ -125,8 +130,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 500 } ) } - - mimeType = userFile.type || 'application/octet-stream' } const maxSize = 250 * 1024 * 1024 diff --git a/apps/sim/app/api/tools/outlook/draft/route.ts b/apps/sim/app/api/tools/outlook/draft/route.ts index 693f0fa2ad5..979634ffcf3 100644 --- a/apps/sim/app/api/tools/outlook/draft/route.ts +++ b/apps/sim/app/api/tools/outlook/draft/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -107,27 +108,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = accessResults.find((r) => r !== null) if (denied) return denied - const buffers = await Promise.all( - attachments.map(async (file) => { - try { + let resolved: Array<{ buffer: Buffer; contentType: string }> + try { + resolved = await Promise.all( + attachments.map(async (file) => { logger.info( `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` ) - return await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - }) - ) + return await downloadServableFileFromStorage(file, requestId, logger) + }) + ) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download an attachment:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + + const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) + if (resolvedTotal > maxSize) { + const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`, + }, + { status: 400 } + ) + } const attachmentObjects = attachments.map((file, i) => ({ '@odata.type': '#microsoft.graph.fileAttachment', name: file.name, - contentType: file.type || 'application/octet-stream', - contentBytes: buffers[i].toString('base64'), + contentType: resolved[i].contentType || file.type || 'application/octet-stream', + contentBytes: resolved[i].buffer.toString('base64'), })) logger.info(`[${requestId}] Converted ${attachmentObjects.length} attachments to base64`) diff --git a/apps/sim/app/api/tools/outlook/send/route.ts b/apps/sim/app/api/tools/outlook/send/route.ts index 080d9db4871..7a59f44410a 100644 --- a/apps/sim/app/api/tools/outlook/send/route.ts +++ b/apps/sim/app/api/tools/outlook/send/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -107,27 +108,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = accessResults.find((r) => r !== null) if (denied) return denied - const buffers = await Promise.all( - attachments.map(async (file) => { - try { + let resolved: Array<{ buffer: Buffer; contentType: string }> + try { + resolved = await Promise.all( + attachments.map(async (file) => { logger.info( `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` ) - return await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - }) - ) + return await downloadServableFileFromStorage(file, requestId, logger) + }) + ) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download an attachment:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + + const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) + if (resolvedTotal > maxSize) { + const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`, + }, + { status: 400 } + ) + } const attachmentObjects = attachments.map((file, i) => ({ '@odata.type': '#microsoft.graph.fileAttachment', name: file.name, - contentType: file.type || 'application/octet-stream', - contentBytes: buffers[i].toString('base64'), + contentType: resolved[i].contentType || file.type || 'application/octet-stream', + contentBytes: resolved[i].buffer.toString('base64'), })) logger.info(`[${requestId}] Converted ${attachmentObjects.length} attachments to base64`) diff --git a/apps/sim/app/api/tools/persona/import-accounts/route.ts b/apps/sim/app/api/tools/persona/import-accounts/route.ts index 8837881b990..0844b11c16a 100644 --- a/apps/sim/app/api/tools/persona/import-accounts/route.ts +++ b/apps/sim/app/api/tools/persona/import-accounts/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess, FileAccessDeniedError } from '@/app/api/files/authorization' import { buildPersonaHeaders, @@ -55,7 +56,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + let buffer: Buffer + try { + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + buffer = resolved.buffer + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download Persona import file:`, error) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status: 500 } + ) + } logger.info(`[${requestId}] Importing accounts into Persona`, { fileName: userFile.name, diff --git a/apps/sim/app/api/tools/s3/put-object/route.ts b/apps/sim/app/api/tools/s3/put-object/route.ts index 13d543aca0a..2019de2b4d4 100644 --- a/apps/sim/app/api/tools/s3/put-object/route.ts +++ b/apps/sim/app/api/tools/s3/put-object/route.ts @@ -8,7 +8,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -81,10 +82,25 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + let downloadedContentType = '' + try { + const result = await downloadServableFileFromStorage(userFile, requestId, logger) + uploadBody = result.buffer + downloadedContentType = result.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to download file') }, + { status: 500 } + ) + } - uploadBody = buffer - uploadContentType = validatedData.contentType || userFile.type || 'application/octet-stream' + uploadContentType = + validatedData.contentType || + downloadedContentType || + userFile.type || + 'application/octet-stream' } else if (validatedData.content) { uploadBody = Buffer.from(validatedData.content, 'utf-8') uploadContentType = validatedData.contentType || 'text/plain' diff --git a/apps/sim/app/api/tools/sap_concur/upload/route.ts b/apps/sim/app/api/tools/sap_concur/upload/route.ts index 74f8fb093de..4a682b3c5d7 100644 --- a/apps/sim/app/api/tools/sap_concur/upload/route.ts +++ b/apps/sim/app/api/tools/sap_concur/upload/route.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' +import { getErrorMessage, toError } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { getValidationErrorMessage, isZodError } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' @@ -7,7 +7,8 @@ import { secureFetchWithValidation } from '@/lib/core/security/input-validation. import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { assertSafeExternalUrl, @@ -208,9 +209,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const userFile = userFiles[0] const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + let fileBuffer: Buffer + let resolvedContentType: string + try { + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = resolved.buffer + resolvedContentType = resolved.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download Concur receipt file:`, error) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } const fileName = userFile.name - const mimeType = inferMimeType(fileName, userFile.type) + const mimeType = inferMimeType(fileName, resolvedContentType || userFile.type) const allowedForOperation = uploadReq.operation === 'create_quick_expense_with_image' diff --git a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts index b70144b1956..d9b2b97876a 100644 --- a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts +++ b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -106,26 +107,46 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = accessResults.find((r) => r !== null) if (denied) return denied - const buffers = await Promise.all( - userFiles.map(async (file) => { - try { + let resolved: Array<{ buffer: Buffer; contentType: string }> + try { + resolved = await Promise.all( + userFiles.map(async (file) => { logger.info( `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` ) - return await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - }) - ) + return await downloadServableFileFromStorage(file, requestId, logger) + }) + ) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download an attachment:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + + const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) + const maxSize = 30 * 1024 * 1024 + if (resolvedTotal > maxSize) { + const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`, + }, + { status: 400 } + ) + } const sendGridAttachments = userFiles.map((file, i) => ({ - content: buffers[i].toString('base64'), + content: resolved[i].buffer.toString('base64'), filename: file.name, - type: file.type || 'application/octet-stream', + type: resolved[i].contentType || file.type || 'application/octet-stream', disposition: 'attachment', })) diff --git a/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts b/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts index 4ddd8d162fb..78dcdb0883b 100644 --- a/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts +++ b/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts @@ -8,7 +8,8 @@ import { secureFetchWithValidation } from '@/lib/core/security/input-validation. import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import type { ServiceNowAttachment } from '@/tools/servicenow/types' import { createBasicAuthHeader } from '@/tools/servicenow/utils' @@ -52,8 +53,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const contentType = userFile.type || 'application/octet-stream' - const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + let fileBuffer: Buffer + let resolvedContentType: string + try { + const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = servable.buffer + resolvedContentType = servable.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download file from storage:`, error) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to download file') }, + { status: 500 } + ) + } + + const contentType = resolvedContentType || userFile.type || 'application/octet-stream' const baseUrl = body.instanceUrl.trim().replace(/\/$/, '') const uploadParams = new URLSearchParams({ diff --git a/apps/sim/app/api/tools/sftp/upload/route.ts b/apps/sim/app/api/tools/sftp/upload/route.ts index 6b686d57c2b..09029d5cd81 100644 --- a/apps/sim/app/api/tools/sftp/upload/route.ts +++ b/apps/sim/app/api/tools/sftp/upload/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { createSftpConnection, @@ -95,6 +96,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } + let resolvedTotal = 0 for (const file of userFiles) { try { const denied = await assertToolFileAccess( @@ -107,7 +109,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.info( `[${requestId}] Downloading file for upload: ${file.name} (${file.size} bytes)` ) - const buffer = await downloadFileFromStorage(file, requestId, logger) + const { buffer } = await downloadServableFileFromStorage(file, requestId, logger) + + resolvedTotal += buffer.length + if (resolvedTotal > maxSize) { + const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` }, + { status: 400 } + ) + } const safeFileName = sanitizeFileName(file.name) const fullRemotePath = remotePath.endsWith('/') @@ -142,6 +153,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger.info(`[${requestId}] Uploaded ${safeFileName} to ${sanitizedRemotePath}`) } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Failed to upload file ${file.name}:`, error) throw new Error( `Failed to upload file "${file.name}": ${getErrorMessage(error, 'Unknown error')}` diff --git a/apps/sim/app/api/tools/sharepoint/upload/route.ts b/apps/sim/app/api/tools/sharepoint/upload/route.ts index af975058eb1..3c85a7bd7d1 100644 --- a/apps/sim/app/api/tools/sharepoint/upload/route.ts +++ b/apps/sim/app/api/tools/sharepoint/upload/route.ts @@ -8,7 +8,8 @@ import { secureFetchWithValidation } from '@/lib/core/security/input-validation. import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' import type { SharepointSkippedFile, SharepointUploadError } from '@/tools/sharepoint/types' @@ -87,7 +88,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (denied) return denied logger.info(`[${requestId}] Uploading file: ${userFile.name}`) - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + let buffer: Buffer + let downloadedContentType = '' + try { + const result = await downloadServableFileFromStorage(userFile, requestId, logger) + buffer = result.buffer + downloadedContentType = result.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + throw error + } const fileName = validatedData.fileName || userFile.name const folderPath = validatedData.folderPath?.trim() || '' @@ -135,7 +146,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { method: 'PUT', headers: { Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': userFile.type || 'application/octet-stream', + 'Content-Type': downloadedContentType || userFile.type || 'application/octet-stream', }, body: buffer, }, @@ -156,7 +167,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { method: 'PUT', headers: { Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': userFile.type || 'application/octet-stream', + 'Content-Type': + downloadedContentType || userFile.type || 'application/octet-stream', }, body: buffer, }, diff --git a/apps/sim/app/api/tools/slack/send-message/route.ts b/apps/sim/app/api/tools/slack/send-message/route.ts index a407df3bbb6..8a972fd4c85 100644 --- a/apps/sim/app/api/tools/slack/send-message/route.ts +++ b/apps/sim/app/api/tools/slack/send-message/route.ts @@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { FileAccessDeniedError } from '@/app/api/files/authorization' import { sendSlackMessage } from '@/app/api/tools/slack/utils' @@ -72,6 +73,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (error instanceof FileAccessDeniedError) { return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) } + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Error sending Slack message:`, error) return NextResponse.json( { diff --git a/apps/sim/app/api/tools/slack/utils.ts b/apps/sim/app/api/tools/slack/utils.ts index 91b6fc14534..b3ff2205806 100644 --- a/apps/sim/app/api/tools/slack/utils.ts +++ b/apps/sim/app/api/tools/slack/utils.ts @@ -1,7 +1,7 @@ import type { Logger } from '@sim/logger' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization' import type { ToolFileData } from '@/tools/types' @@ -89,7 +89,11 @@ async function uploadFilesToSlack( throw new FileAccessDeniedError() } - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger + ) const getUrlResponse = await fetch('https://slack.com/api/files.getUploadURLExternal', { method: 'POST', @@ -131,7 +135,7 @@ async function uploadFilesToSlack( // Only add to uploadedFiles after successful upload to keep arrays in sync uploadedFiles.push({ name: userFile.name, - mimeType: userFile.type || 'application/octet-stream', + mimeType: contentType || userFile.type || 'application/octet-stream', data: buffer.toString('base64'), size: buffer.length, }) diff --git a/apps/sim/app/api/tools/smtp/send/route.ts b/apps/sim/app/api/tools/smtp/send/route.ts index e08101c8596..be1c4371fb6 100644 --- a/apps/sim/app/api/tools/smtp/send/route.ts +++ b/apps/sim/app/api/tools/smtp/send/route.ts @@ -9,7 +9,8 @@ import { validateDatabaseHost } from '@/lib/core/security/input-validation.serve import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -127,26 +128,45 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = accessResults.find((r) => r !== null) if (denied) return denied - const buffers = await Promise.all( - attachments.map(async (file) => { - try { + let resolved: Array<{ buffer: Buffer; contentType: string }> + try { + resolved = await Promise.all( + attachments.map(async (file) => { logger.info( `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` ) - return await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - }) - ) + return await downloadServableFileFromStorage(file, requestId, logger) + }) + ) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download an attachment:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + + const resolvedTotal = resolved.reduce((sum, r) => sum + r.buffer.length, 0) + if (resolvedTotal > maxSize) { + const sizeMB = (resolvedTotal / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`, + }, + { status: 400 } + ) + } const attachmentBuffers = attachments.map((file, i) => ({ filename: file.name, - content: buffers[i], - contentType: file.type || 'application/octet-stream', + content: resolved[i].buffer, + contentType: resolved[i].contentType || file.type || 'application/octet-stream', })) logger.info(`[${requestId}] Processed ${attachmentBuffers.length} attachment(s)`) diff --git a/apps/sim/app/api/tools/supabase/storage-upload/route.ts b/apps/sim/app/api/tools/supabase/storage-upload/route.ts index 9a3a2643c58..3e2dcd4cdd6 100644 --- a/apps/sim/app/api/tools/supabase/storage-upload/route.ts +++ b/apps/sim/app/api/tools/supabase/storage-upload/route.ts @@ -8,7 +8,8 @@ import { validateSupabaseProjectId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -147,10 +148,28 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + let buffer: Buffer + let resolvedContentType: string + try { + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + buffer = resolved.buffer + resolvedContentType = resolved.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download file for Supabase upload:`, error) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status: 500 } + ) + } uploadBody = buffer - uploadContentType = validatedData.contentType || userFile.type || 'application/octet-stream' + uploadContentType = + validatedData.contentType || + resolvedContentType || + userFile.type || + 'application/octet-stream' } let fullPath = validatedData.fileName diff --git a/apps/sim/app/api/tools/telegram/send-document/route.ts b/apps/sim/app/api/tools/telegram/send-document/route.ts index 2d685c501fd..f454717e290 100644 --- a/apps/sim/app/api/tools/telegram/send-document/route.ts +++ b/apps/sim/app/api/tools/telegram/send-document/route.ts @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { convertMarkdownToHTML } from '@/tools/telegram/utils' @@ -93,11 +94,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + let buffer: Buffer + let contentType: string + try { + const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger) + buffer = downloaded.buffer + contentType = downloaded.contentType + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download document ${userFile.name}:`, error) + return NextResponse.json( + { + success: false, + error: `Failed to download document: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + + if (buffer.length > maxSize) { + const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2) + return NextResponse.json( + { + success: false, + error: `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`, + }, + { status: 400 } + ) + } + + const resolvedMimeType = contentType || userFile.type || 'application/octet-stream' const filesOutput = [ { name: userFile.name, - mimeType: userFile.type || 'application/octet-stream', + mimeType: resolvedMimeType, data: buffer.toString('base64'), size: buffer.length, }, @@ -108,7 +139,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const formData = new FormData() formData.append('chat_id', validatedData.chatId) - const blob = new Blob([new Uint8Array(buffer)], { type: userFile.type }) + const blob = new Blob([new Uint8Array(buffer)], { type: resolvedMimeType }) formData.append('document', blob, userFile.name) if (validatedData.caption) { diff --git a/apps/sim/app/api/tools/textract/parse/route.ts b/apps/sim/app/api/tools/textract/parse/route.ts index b93cbbed4d9..82eb65ff830 100644 --- a/apps/sim/app/api/tools/textract/parse/route.ts +++ b/apps/sim/app/api/tools/textract/parse/route.ts @@ -16,9 +16,10 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { isInternalFileUrl, processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' import { - downloadFileFromStorage, + downloadServableFileFromStorage, resolveInternalFileUrl, } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -437,9 +438,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger) + const { buffer, contentType: resolvedContentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger + ) bytes = buffer.toString('base64') - contentType = userFile.type || 'application/octet-stream' + contentType = resolvedContentType || userFile.type || 'application/octet-stream' isPdf = contentType.includes('pdf') || userFile.name?.toLowerCase().endsWith('.pdf') } else if (validatedData.filePath) { let fileUrl = validatedData.filePath @@ -616,6 +621,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }, }) } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Error in Textract parse:`, error) return NextResponse.json( diff --git a/apps/sim/app/api/tools/uptimerobot/create-psp/route.ts b/apps/sim/app/api/tools/uptimerobot/create-psp/route.ts index 0b0036711ec..1d879c5735b 100644 --- a/apps/sim/app/api/tools/uptimerobot/create-psp/route.ts +++ b/apps/sim/app/api/tools/uptimerobot/create-psp/route.ts @@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { forwardPspRequest } from '@/app/api/tools/uptimerobot/server-utils' export const dynamic = 'force-dynamic' @@ -39,6 +40,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger, }) } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Unexpected error creating status page:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Unknown error') }, diff --git a/apps/sim/app/api/tools/uptimerobot/server-utils.ts b/apps/sim/app/api/tools/uptimerobot/server-utils.ts index f94d66c0948..d7a1ef9ea8c 100644 --- a/apps/sim/app/api/tools/uptimerobot/server-utils.ts +++ b/apps/sim/app/api/tools/uptimerobot/server-utils.ts @@ -1,7 +1,7 @@ import type { Logger } from '@sim/logger' import { NextResponse } from 'next/server' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' import { mapPsp, UPTIMEROBOT_API_BASE } from '@/tools/uptimerobot/types' @@ -47,8 +47,8 @@ async function appendPspImage( const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) if (denied) return denied - const buffer = await downloadFileFromStorage(userFile, requestId, logger) - const mimeType = userFile.type || 'application/octet-stream' + const { buffer, contentType } = await downloadServableFileFromStorage(userFile, requestId, logger) + const mimeType = contentType || userFile.type || 'application/octet-stream' form.append(field, new Blob([new Uint8Array(buffer)], { type: mimeType }), userFile.name) return null } diff --git a/apps/sim/app/api/tools/uptimerobot/update-psp/route.ts b/apps/sim/app/api/tools/uptimerobot/update-psp/route.ts index a3787686b9a..063a4252ca6 100644 --- a/apps/sim/app/api/tools/uptimerobot/update-psp/route.ts +++ b/apps/sim/app/api/tools/uptimerobot/update-psp/route.ts @@ -6,6 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { forwardPspRequest } from '@/app/api/tools/uptimerobot/server-utils' export const dynamic = 'force-dynamic' @@ -39,6 +40,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { logger, }) } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Unexpected error updating status page:`, error) return NextResponse.json( { success: false, error: getErrorMessage(error, 'Unknown error') }, diff --git a/apps/sim/app/api/tools/vanta/upload/route.ts b/apps/sim/app/api/tools/vanta/upload/route.ts index a76cc848c01..81785b515c8 100644 --- a/apps/sim/app/api/tools/vanta/upload/route.ts +++ b/apps/sim/app/api/tools/vanta/upload/route.ts @@ -1,5 +1,5 @@ import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' +import { getErrorMessage, toError } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { vantaUploadContract } from '@/lib/api/contracts/tools/vanta' import { parseRequest } from '@/lib/api/server' @@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { asVantaRecord, @@ -70,9 +71,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return uploadSizeError(userFile.size) } - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) - fileName = params.fileName || userFile.name - mimeType = userFile.type || params.mimeType || 'application/octet-stream' + try { + const resolved = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = resolved.buffer + fileName = params.fileName || userFile.name + mimeType = + resolved.contentType || userFile.type || params.mimeType || 'application/octet-stream' + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${requestId}] Failed to download Vanta upload file`, { + error: getErrorMessage(error), + }) + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to download file') }, + { status: 500 } + ) + } } else if (params.fileContent) { fileBuffer = Buffer.from(params.fileContent, 'base64') fileName = params.fileName || 'file' diff --git a/apps/sim/app/api/tools/wordpress/upload/route.ts b/apps/sim/app/api/tools/wordpress/upload/route.ts index a62632caea6..aac9e9e3354 100644 --- a/apps/sim/app/api/tools/wordpress/upload/route.ts +++ b/apps/sim/app/api/tools/wordpress/upload/route.ts @@ -11,7 +11,8 @@ import { getMimeTypeFromExtension, processSingleFileToUserFile, } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -89,10 +90,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }) let fileBuffer: Buffer + let resolvedContentType: string try { - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) + const servable = await downloadServableFileFromStorage(userFile, requestId, logger) + fileBuffer = servable.buffer + resolvedContentType = servable.contentType } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady logger.error(`[${requestId}] Failed to download file:`, error) return NextResponse.json( { @@ -104,7 +110,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const filename = validatedData.filename || userFile.name - const mimeType = userFile.type || getMimeTypeFromExtension(getFileExtension(filename)) + const mimeType = + resolvedContentType || userFile.type || getMimeTypeFromExtension(getFileExtension(filename)) logger.info(`[${requestId}] Uploading to WordPress`, { siteId: validatedData.siteId, diff --git a/apps/sim/lib/copilot/tools/server/files/doc-compile.ts b/apps/sim/lib/copilot/tools/server/files/doc-compile.ts index 1429fc705d1..e27e94734d9 100644 --- a/apps/sim/lib/copilot/tools/server/files/doc-compile.ts +++ b/apps/sim/lib/copilot/tools/server/files/doc-compile.ts @@ -1,11 +1,16 @@ import { createLogger } from '@sim/logger' +import { sha256Hex } from '@sim/security/hash' +import { isE2BDocEnabled } from '@/lib/core/config/env-flags' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { executeInE2B, executeShellInE2B, type SandboxFile } from '@/lib/execution/e2b' import { CodeLanguage } from '@/lib/execution/languages' +import { runSandboxTask } from '@/lib/execution/sandbox/run-task' import { fetchWorkspaceFileBuffer, getWorkspaceFile, } from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { getContentType } from '@/app/api/files/utils' +import type { SandboxTaskId } from '@/sandbox-tasks/registry' import { loadCompiledDoc, storeCompiledDoc } from './doc-compiled-store' const logger = createLogger('CopilotDocCompile') @@ -449,3 +454,102 @@ export async function resolveServableDoc( const artifact = await loadCompiledDocByExt(workspaceId, storedBytes.toString('utf-8'), fmt.ext) return artifact ? { kind: 'artifact', ...artifact } : { kind: 'unavailable' } } + +interface CompilableFormat { + magic: Buffer + taskId: SandboxTaskId + contentType: string +} + +const COMPILABLE_FORMATS: Record = { + '.pptx': { magic: ZIP_MAGIC, taskId: 'pptx-generate', contentType: PPTX_MIME }, + '.docx': { magic: ZIP_MAGIC, taskId: 'docx-generate', contentType: DOCX_MIME }, + '.pdf': { magic: PDF_MAGIC, taskId: 'pdf-generate', contentType: PDF_MIME }, +} + +const MAX_COMPILED_DOC_CACHE = 10 +const compiledDocCache = new Map() + +function compiledCacheSet(key: string, buffer: Buffer): void { + if (compiledDocCache.size >= MAX_COMPILED_DOC_CACHE) { + compiledDocCache.delete(compiledDocCache.keys().next().value as string) + } + compiledDocCache.set(key, buffer) +} + +/** + * Resolves the bytes a consumer should actually serve/attach for a stored file — + * the single source of truth shared by the file-serve route and every tool that + * downloads a workspace file (email attachments, uploads, provider file inputs). + * + * Generated docs (pdf/docx/pptx/xlsx) store their GENERATION SOURCE as the primary + * file; the rendered binary lives in a separate content-addressed artifact store. + * A naive raw-byte read therefore hands out source text under a `.pdf` name — the + * corruption every non-serve consumer used to ship. The file-serve route and the + * attachment download helper share this one function so they resolve identically. + * (The public read-only share route uses the non-compiling {@link resolveServableDoc} + * variant, which returns `unavailable` instead of throwing.) The swap: + * + * - Bytes already carry the format magic (`%PDF`/ZIP) → real uploaded/binary file, + * serve as-is. + * - Generated-doc source → load the content-addressed compiled artifact. + * - Artifact missing in the E2B regime → the doc is still being generated; throw + * {@link DocCompileUserError} so callers signal "not ready / retry" instead of + * shipping source. + * - E2B disabled → compile the committed JS source via isolated-vm (cached). + * - Non-doc files → pass through with the extension-derived content type. + * + * It never falls back to attaching the raw source bytes for a generated doc. + */ +export async function resolveServableDocBytes(args: { + rawBuffer: Buffer + fileName: string + workspaceId: string | undefined + ownerKey?: string + signal?: AbortSignal +}): Promise<{ buffer: Buffer; contentType: string }> { + const { rawBuffer, fileName, workspaceId, ownerKey, signal } = args + const ext = fileName.slice(fileName.lastIndexOf('.')).toLowerCase() + const extNoDot = ext.replace(/^\./, '') + const format = COMPILABLE_FORMATS[ext] + + // xlsx isn't in COMPILABLE_FORMATS (no isolated-vm path), so match its ZIP magic + // explicitly alongside the table-driven formats. + const magic = format?.magic ?? (extNoDot === 'xlsx' ? ZIP_MAGIC : undefined) + if (magic && bufferStartsWith(rawBuffer, magic)) { + return { buffer: rawBuffer, contentType: getContentType(fileName) } + } + + if (!format && extNoDot !== 'xlsx') { + return { buffer: rawBuffer, contentType: getContentType(fileName) } + } + + const source = rawBuffer.toString('utf-8') + + if (workspaceId) { + const stored = await loadCompiledDocByExt(workspaceId, source, extNoDot) + if (stored) { + return { buffer: stored.buffer, contentType: stored.contentType } + } + if (isE2BDocEnabled && (await getE2BDocFormat(fileName))) { + throw new DocCompileUserError('Document is still being generated') + } + } + + // Reaches here only for xlsx, which has no isolated-vm fallback. + if (!format) return { buffer: rawBuffer, contentType: getContentType(fileName) } + + const cacheKey = sha256Hex(`${ext}${source}${workspaceId ?? ''}`) + const cached = compiledDocCache.get(cacheKey) + if (cached) { + return { buffer: cached, contentType: format.contentType } + } + + const compiled = await runSandboxTask( + format.taskId, + { code: source, workspaceId: workspaceId || '' }, + { ownerKey, signal } + ) + compiledCacheSet(cacheKey, compiled) + return { buffer: compiled, contentType: format.contentType } +} diff --git a/apps/sim/lib/copilot/tools/server/files/doc-servable.test.ts b/apps/sim/lib/copilot/tools/server/files/doc-servable.test.ts new file mode 100644 index 00000000000..f72695cb539 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/files/doc-servable.test.ts @@ -0,0 +1,183 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { e2bFlag, betaFlag, mockLoadCompiledDoc, mockRunSandboxTask } = vi.hoisted(() => ({ + e2bFlag: { value: true }, + betaFlag: { value: false }, + mockLoadCompiledDoc: vi.fn(), + mockRunSandboxTask: vi.fn(), +})) + +vi.mock('@/lib/execution/e2b', () => ({ + executeInE2B: vi.fn(), + executeShellInE2B: vi.fn(), +})) +vi.mock('@/lib/execution/languages', () => ({ + CodeLanguage: { javascript: 'javascript', python: 'python' }, +})) +vi.mock('@/lib/execution/sandbox/run-task', () => ({ + runSandboxTask: mockRunSandboxTask, +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + getWorkspaceFile: vi.fn(), + fetchWorkspaceFileBuffer: vi.fn(), +})) +vi.mock('./doc-compiled-store', () => ({ + loadCompiledDoc: mockLoadCompiledDoc, + storeCompiledDoc: vi.fn(), +})) +vi.mock('@/lib/core/config/feature-flags', () => ({ + isFeatureEnabled: vi.fn(async () => betaFlag.value), +})) +vi.mock('@/lib/core/config/env-flags', () => ({ + get isE2BDocEnabled() { + return e2bFlag.value + }, +})) +vi.mock('@/app/api/files/utils', () => ({ + getContentType: (name: string) => + name.endsWith('.pdf') + ? 'application/pdf' + : name.endsWith('.txt') + ? 'text/plain' + : 'application/octet-stream', +})) + +import { DocCompileUserError, resolveServableDocBytes } from './doc-compile' + +const WORKSPACE_ID = '550e8400-e29b-41d4-a716-446655440000' +const PDF_MAGIC = Buffer.from('%PDF-1.7\n...binary...') +const PDF_SOURCE = Buffer.from('from reportlab.pdfgen import canvas\n# generates a PDF', 'utf-8') +const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x00, 0x01]) +const XLSX_SOURCE = Buffer.from('from openpyxl import Workbook\n# generates an xlsx', 'utf-8') + +describe('resolveServableDocBytes', () => { + beforeEach(() => { + vi.clearAllMocks() + e2bFlag.value = true + betaFlag.value = false + }) + + it('swaps generated-doc source for the compiled artifact + binary content type', async () => { + const artifact = Buffer.from('%PDF-compiled-binary') + mockLoadCompiledDoc.mockResolvedValue(artifact) + + const result = await resolveServableDocBytes({ + rawBuffer: PDF_SOURCE, + fileName: 'report.pdf', + workspaceId: WORKSPACE_ID, + }) + + expect(result.buffer).toBe(artifact) + expect(result.contentType).toBe('application/pdf') + expect(mockLoadCompiledDoc).toHaveBeenCalledWith( + WORKSPACE_ID, + PDF_SOURCE.toString('utf-8'), + 'pdf' + ) + }) + + it('passes through a real binary PDF (carries the %PDF magic) without an artifact lookup', async () => { + const result = await resolveServableDocBytes({ + rawBuffer: PDF_MAGIC, + fileName: 'uploaded.pdf', + workspaceId: WORKSPACE_ID, + }) + + expect(result.buffer).toBe(PDF_MAGIC) + expect(result.contentType).toBe('application/pdf') + expect(mockLoadCompiledDoc).not.toHaveBeenCalled() + }) + + it('throws DocCompileUserError when a generated doc artifact is not ready (E2B regime)', async () => { + mockLoadCompiledDoc.mockResolvedValue(null) + e2bFlag.value = true + + await expect( + resolveServableDocBytes({ + rawBuffer: PDF_SOURCE, + fileName: 'report.pdf', + workspaceId: WORKSPACE_ID, + }) + ).rejects.toBeInstanceOf(DocCompileUserError) + + expect(mockRunSandboxTask).not.toHaveBeenCalled() + }) + + it('compiles via the sandbox when E2B is disabled and no artifact is stored', async () => { + mockLoadCompiledDoc.mockResolvedValue(null) + e2bFlag.value = false + const compiled = Buffer.from('%PDF-isolated-vm-binary') + mockRunSandboxTask.mockResolvedValue(compiled) + + const result = await resolveServableDocBytes({ + rawBuffer: PDF_SOURCE, + fileName: 'report.pdf', + workspaceId: WORKSPACE_ID, + }) + + expect(result.buffer).toBe(compiled) + expect(result.contentType).toBe('application/pdf') + expect(mockRunSandboxTask).toHaveBeenCalledWith( + 'pdf-generate', + { code: PDF_SOURCE.toString('utf-8'), workspaceId: WORKSPACE_ID }, + expect.objectContaining({}) + ) + }) + + it('passes non-doc files through untouched with their extension content type', async () => { + const text = Buffer.from('hello world', 'utf-8') + const result = await resolveServableDocBytes({ + rawBuffer: text, + fileName: 'notes.txt', + workspaceId: WORKSPACE_ID, + }) + + expect(result.buffer).toBe(text) + expect(result.contentType).toBe('text/plain') + expect(mockLoadCompiledDoc).not.toHaveBeenCalled() + }) + + it('passes through a real binary XLSX (ZIP magic) without an artifact lookup', async () => { + const result = await resolveServableDocBytes({ + rawBuffer: ZIP_MAGIC, + fileName: 'sheet.xlsx', + workspaceId: WORKSPACE_ID, + }) + + expect(result.buffer).toBe(ZIP_MAGIC) + expect(mockLoadCompiledDoc).not.toHaveBeenCalled() + }) + + it('throws when a generated XLSX artifact is not ready (E2B + mothership-beta enabled)', async () => { + mockLoadCompiledDoc.mockResolvedValue(null) + e2bFlag.value = true + betaFlag.value = true + + await expect( + resolveServableDocBytes({ + rawBuffer: XLSX_SOURCE, + fileName: 'sheet.xlsx', + workspaceId: WORKSPACE_ID, + }) + ).rejects.toBeInstanceOf(DocCompileUserError) + + expect(mockRunSandboxTask).not.toHaveBeenCalled() + }) + + it('returns raw XLSX source when there is no workspaceId (xlsx has no isolated-vm path)', async () => { + betaFlag.value = true + + const result = await resolveServableDocBytes({ + rawBuffer: XLSX_SOURCE, + fileName: 'sheet.xlsx', + workspaceId: undefined, + }) + + expect(result.buffer).toBe(XLSX_SOURCE) + expect(mockLoadCompiledDoc).not.toHaveBeenCalled() + expect(mockRunSandboxTask).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index b5460c8440a..e495b94274a 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -17,6 +17,8 @@ import { StorageService } from '@/lib/uploads' import { isExecutionFile } from '@/lib/uploads/contexts/execution/utils' import { extractStorageKey, + getFileExtension, + getMimeTypeFromExtension, inferContextFromKey, isInternalFileUrl, processSingleFileToUserFile, @@ -300,3 +302,68 @@ export async function downloadFileFromStorage( return buffer } + +/** + * Result of {@link downloadServableFileFromStorage}: the bytes a consumer should + * actually attach/upload, plus the content type that matches those bytes. + */ +export interface ServableFile { + buffer: Buffer + contentType: string +} + +/** + * Downloads a workspace file and resolves it to its SERVABLE bytes — the variant + * every tool that hands a file to an external service (email attachments, chat + * uploads, provider file inputs) should use instead of {@link downloadFileFromStorage}. + * + * AI-generated docs (pdf/docx/pptx/xlsx) store their generation SOURCE as the + * primary file and keep the rendered binary in a separate content-addressed + * artifact store. A raw download therefore yields source text under a `.pdf` + * name — the file the recipient cannot open. This swaps in the compiled artifact + * (and the correct binary content type) via the same resolver the file-serve + * route uses, so the serve and attachment paths resolve identically. Non-doc files + * and real uploaded binaries pass through unchanged, carrying `userFile.type` when set. + * + * Throws `DocCompileUserError` when a generated doc's artifact is not ready (still + * compiling) — callers should surface a retryable error rather than attach source. + */ +export async function downloadServableFileFromStorage( + userFile: UserFile, + requestId: string, + logger: Logger, + options: { maxBytes?: number; signal?: AbortSignal; ownerKey?: string } = {} +): Promise { + const buffer = await downloadFileFromStorage(userFile, requestId, logger, { + maxBytes: options.maxBytes, + }) + + // Cheap pre-filter so only generated-doc candidates pay for the heavier resolver + // import below. + const ext = getFileExtension(userFile.name) + if (ext !== 'pdf' && ext !== 'docx' && ext !== 'pptx' && ext !== 'xlsx') { + return { buffer, contentType: userFile.type || getMimeTypeFromExtension(ext) } + } + + const { parseWorkspaceFileKey } = await import( + '@/lib/uploads/contexts/workspace/workspace-file-manager' + ) + const workspaceId = userFile.key ? (parseWorkspaceFileKey(userFile.key) ?? undefined) : undefined + + const { resolveServableDocBytes } = await import('@/lib/copilot/tools/server/files/doc-compile') + const resolved = await resolveServableDocBytes({ + rawBuffer: buffer, + fileName: userFile.name, + workspaceId, + ownerKey: options.ownerKey, + signal: options.signal, + }) + + // Re-check: the raw download enforced maxBytes on the source, but a generated doc + // resolves to a larger artifact. + if (options.maxBytes !== undefined && resolved.buffer.length > options.maxBytes) { + assertKnownSizeWithinLimit(resolved.buffer.length, options.maxBytes, 'servable file download') + } + + return resolved +} diff --git a/apps/sim/lib/uploads/utils/servable-file-response.ts b/apps/sim/lib/uploads/utils/servable-file-response.ts new file mode 100644 index 00000000000..1e7d1a6124b --- /dev/null +++ b/apps/sim/lib/uploads/utils/servable-file-response.ts @@ -0,0 +1,23 @@ +import { NextResponse } from 'next/server' +import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile' + +/** + * Canonical retryable response for an attachment/upload whose generated-document + * artifact is still compiling. Returns the 409 when `error` is a + * {@link DocCompileUserError} (thrown by `downloadServableFileFromStorage`), + * otherwise `null` so the caller falls through to its own error handling. Shared + * by every tool route that downloads workspace files so the status, body shape, + * and user-facing copy stay identical instead of being re-typed per route. + */ +export function docNotReadyResponse(error: unknown): NextResponse | null { + if (error instanceof DocCompileUserError) { + return NextResponse.json( + { + success: false, + error: 'A document is still being generated. Wait for it to finish, then try again.', + }, + { status: 409 } + ) + } + return null +} diff --git a/apps/sim/providers/file-attachments.server.ts b/apps/sim/providers/file-attachments.server.ts index 50b48a95465..8f2e2dfac5e 100644 --- a/apps/sim/providers/file-attachments.server.ts +++ b/apps/sim/providers/file-attachments.server.ts @@ -5,7 +5,7 @@ import { sleep } from '@sim/utils/helpers' import type { StorageContext } from '@/lib/uploads' import { StorageService } from '@/lib/uploads' import { inferContextFromKey } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { verifyFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' import { @@ -178,8 +178,15 @@ function groupUploadableFiles(messages: Message[] | undefined): UserFile[][] { * object storage works. Bounded by the provider's attachment ceiling. */ async function downloadFileForUpload(file: UserFile, maxBytes: number): Promise { - const buffer = await downloadFileFromStorage(file, 'provider-file-upload', logger, { maxBytes }) - return new Blob([new Uint8Array(buffer)], { type: file.type || inferAttachmentMimeType(file) }) + const { buffer, contentType } = await downloadServableFileFromStorage( + file, + 'provider-file-upload', + logger, + { maxBytes } + ) + return new Blob([new Uint8Array(buffer)], { + type: contentType || file.type || inferAttachmentMimeType(file), + }) } /** diff --git a/apps/sim/tools/microsoft_teams/server-utils.ts b/apps/sim/tools/microsoft_teams/server-utils.ts index c1e7376f6ab..c5558ad8512 100644 --- a/apps/sim/tools/microsoft_teams/server-utils.ts +++ b/apps/sim/tools/microsoft_teams/server-utils.ts @@ -6,7 +6,7 @@ import type { Logger } from '@sim/logger' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' import type { GraphApiErrorResponse, GraphDriveItem } from '@/tools/microsoft_teams/types' @@ -80,10 +80,19 @@ export async function uploadFilesForTeamsMessage(params: { } // Download file from storage - const buffer = await downloadFileFromStorage(file, requestId, log) + const { buffer, contentType } = await downloadServableFileFromStorage(file, requestId, log) + + if (buffer.length > MAX_TEAMS_FILE_SIZE) { + const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2) + throw new Error( + `File "${file.name}" (${sizeMB}MB) exceeds the 4MB limit for Teams attachments. Use smaller files or upload to SharePoint/OneDrive first.` + ) + } + + const resolvedMimeType = contentType || file.type || 'application/octet-stream' filesOutput.push({ name: file.name, - mimeType: file.type || 'application/octet-stream', + mimeType: resolvedMimeType, data: buffer.toString('base64'), size: buffer.length, }) @@ -102,7 +111,7 @@ export async function uploadFilesForTeamsMessage(params: { method: 'PUT', headers: { Authorization: `Bearer ${accessToken}`, - 'Content-Type': file.type || 'application/octet-stream', + 'Content-Type': resolvedMimeType, }, body: buffer, },