fix(uploads): attach compiled binary for AI-generated docs, not source

apps/sim/app/api/files/serve/[...path]/route.ts

@@ -1,18 +1,14 @@
 import { readFile } from 'fs/promises'
 import { createLogger } from '@sim/logger'
-import { sha256Hex } from '@sim/security/hash'
 import type { NextRequest } from 'next/server'
 import { NextResponse } from 'next/server'
 import { fileServeParamsSchema, fileServeQuerySchema } from '@/lib/api/contracts/storage-transfer'
 import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
 import {
   DocCompileUserError,
-  getE2BDocFormat,
-  loadCompiledDocByExt,
+  resolveServableDocBytes,
 } from '@/lib/copilot/tools/server/files/doc-compile'
-import { isE2BDocEnabled } from '@/lib/core/config/env-flags'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
-import { runSandboxTask } from '@/lib/execution/sandbox/run-task'
 import { CopilotFiles, isUsingCloudStorage } from '@/lib/uploads'
 import type { StorageContext } from '@/lib/uploads/config'
 import { parseWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
@@ -26,47 +22,14 @@ import {
   findLocalFile,
   getContentType,
 } from '@/app/api/files/utils'
-import type { SandboxTaskId } from '@/sandbox-tasks/registry'
 
 const logger = createLogger('FilesServeAPI')
 
-const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04])
-const PDF_MAGIC = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d]) // %PDF-
-
-interface CompilableFormat {
-  magic: Buffer
-  taskId: SandboxTaskId
-  contentType: string
-}
-
-const COMPILABLE_FORMATS: Record<string, CompilableFormat> = {
-  '.pptx': {
-    magic: ZIP_MAGIC,
-    taskId: 'pptx-generate',
-    contentType: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-  },
-  '.docx': {
-    magic: ZIP_MAGIC,
-    taskId: 'docx-generate',
-    contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-  },
-  '.pdf': {
-    magic: PDF_MAGIC,
-    taskId: 'pdf-generate',
-    contentType: 'application/pdf',
-  },
-}
-
-const MAX_COMPILED_DOC_CACHE = 10
-const compiledDocCache = new Map<string, Buffer>()
-
-function compiledCacheSet(key: string, buffer: Buffer): void {
-  if (compiledDocCache.size >= MAX_COMPILED_DOC_CACHE) {
-    compiledDocCache.delete(compiledDocCache.keys().next().value as string)
-  }
-  compiledDocCache.set(key, buffer)
-}
-
+/**
+ * Resolves the bytes + content type to serve for a stored file via the shared
+ * {@link resolveServableDocBytes} (generated docs → compiled artifact). `raw=1`
+ * bypasses resolution and serves the stored source as-is.
+ */
 async function compileDocumentIfNeeded(
   buffer: Buffer,
   filename: string,
@@ -76,71 +39,13 @@ async function compileDocumentIfNeeded(
   signal: AbortSignal | undefined
 ): Promise<{ buffer: Buffer; contentType: string }> {
   if (raw) return { buffer, contentType: getContentType(filename) }
-
-  const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase()
-  const extNoDot = ext.replace(/^\./, '')
-  const format = COMPILABLE_FORMATS[ext]
-
-  // Already a binary file (uploaded or pre-compiled)? Serve as-is.
-  if (format) {
-    const magicLen = format.magic.length
-    if (buffer.length >= magicLen && buffer.subarray(0, magicLen).equals(format.magic)) {
-      return { buffer, contentType: getContentType(filename) }
-    }
-  }
-
-  // .xlsx is a ZIP container with no JS compile path. An uploaded/binary xlsx
-  // must short-circuit here (it isn't in COMPILABLE_FORMATS) — otherwise every
-  // xlsx open would utf-8-decode the whole binary and do an always-miss S3 GET.
-  // Only a Python-source xlsx (UTF-8 text, no ZIP magic) falls through.
-  if (
-    extNoDot === 'xlsx' &&
-    buffer.length >= ZIP_MAGIC.length &&
-    buffer.subarray(0, ZIP_MAGIC.length).equals(ZIP_MAGIC)
-  ) {
-    return { buffer, contentType: getContentType(filename) }
-  }
-
-  // Generated docs render from a content-addressed compiled binary that is built
-  // exactly ONCE per edit_content/create (at write time) and stored in S3. Serve
-  // only LOADS it — it must never compile, or it would re-run E2B on every preview
-  // fetch, including against the incomplete source mid-generation. A hit returns
-  // the (possibly partial) committed doc; a miss in the E2B regime means the doc
-  // is still being generated → 409, and the client polls until the artifact lands.
-  if (workspaceId && (format || extNoDot === 'xlsx')) {
-    const source = buffer.toString('utf-8')
-    // Load the prebuilt artifact directly from S3 (content-addressed). No extra
-    // in-memory layer here: the store is the source of truth, the client (react
-    // query) already caches the bytes, and this branch never recomputes.
-    const stored = await loadCompiledDocByExt(workspaceId, source, extNoDot)
-    if (stored) {
-      return { buffer: stored.buffer, contentType: stored.contentType }
-    }
-
-    if (isE2BDocEnabled && (await getE2BDocFormat(filename))) {
-      // Artifact not built yet (still generating, or the source didn't compile at
-      // write time). Signal "not ready" without compiling — handled as 409.
-      throw new DocCompileUserError('Document is still being generated')
-    }
-  }
-
-  if (!format) return { buffer, contentType: getContentType(filename) }
-
-  // E2B disabled and no stored artifact → compile JS source via isolated-vm.
-  const code = buffer.toString('utf-8')
-  const cacheKey = sha256Hex(`${ext}${code}${workspaceId ?? ''}`)
-  const cached = compiledDocCache.get(cacheKey)
-  if (cached) {
-    return { buffer: cached, contentType: format.contentType }
-  }
-
-  const compiled = await runSandboxTask(
-    format.taskId,
-    { code, workspaceId: workspaceId || '' },
-    { ownerKey, signal }
-  )
-  compiledCacheSet(cacheKey, compiled)
-  return { buffer: compiled, contentType: format.contentType }
+  return resolveServableDocBytes({
+    rawBuffer: buffer,
+    fileName: filename,
+    workspaceId,
+    ownerKey,
+    signal,
+  })
 }
 
 const STORAGE_KEY_PREFIX_RE = /^\d{13}-[a-z0-9]{7}-/

apps/sim/app/api/tools/a2a/send-message/route.ts

@@ -17,7 +17,8 @@ import { enforceUserOrIpRateLimit } from '@/lib/core/rate-limiter'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
-import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
 import { assertToolFileAccess } from '@/app/api/files/authorization'
 
 export const dynamic = 'force-dynamic'
@@ -90,13 +91,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
         if (denied) return denied
       }
       files = await Promise.all(
-        userFiles.map(async (userFile) => ({
-          bytes: await downloadFileFromStorage(userFile, requestId, logger, {
-            maxBytes: A2A_MAX_FILE_BYTES,
-          }),
-          name: userFile.name,
-          mediaType: userFile.type || 'application/octet-stream',
-        }))
+        userFiles.map(async (userFile) => {
+          const { buffer, contentType } = await downloadServableFileFromStorage(
+            userFile,
+            requestId,
+            logger,
+            { maxBytes: A2A_MAX_FILE_BYTES }
+          )
+          return {
+            bytes: buffer,
+            name: userFile.name,
+            mediaType: contentType || userFile.type || 'application/octet-stream',
+          }
+        })
       )
     }
 
@@ -130,6 +137,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
       output,
     })
   } catch (error) {
+    const notReady = docNotReadyResponse(error)
+    if (notReady) return notReady
     logger.error(`[${requestId}] A2A send failed`, { error: getErrorMessage(error) })
     return NextResponse.json({ success: false, error: getErrorMessage(error) }, { status: 502 })
   }

apps/sim/app/api/tools/agiloft/attach/route.test.ts

@@ -21,7 +21,7 @@ vi.mock('@/lib/uploads/utils/file-utils', () => ({
   processFilesToUserFiles: mockProcessFilesToUserFiles,
 }))
 vi.mock('@/lib/uploads/utils/file-utils.server', () => ({
-  downloadFileFromStorage: mockDownloadFileFromStorage,
+  downloadServableFileFromStorage: mockDownloadFileFromStorage,
 }))
 vi.mock('@/app/api/files/authorization', () => ({
   assertToolFileAccess: mockAssertToolFileAccess,
@@ -77,7 +77,10 @@ beforeEach(() => {
     { key: 's3://bucket/file.txt', name: 'file.txt', size: 5, type: 'text/plain' },
   ])
   mockAssertToolFileAccess.mockResolvedValue(null)
-  mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('hello'))
+  mockDownloadFileFromStorage.mockResolvedValue({
+    buffer: Buffer.from('hello'),
+    contentType: 'application/octet-stream',
+  })
 })
 
 describe('POST /api/tools/agiloft/attach', () => {

apps/sim/app/api/tools/agiloft/attach/route.ts

@@ -9,7 +9,8 @@ import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import type { RawFileInput } from '@/lib/uploads/utils/file-schemas'
 import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils'
-import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
 import { assertToolFileAccess } from '@/app/api/files/authorization'
 import { buildAttachFileUrl } from '@/tools/agiloft/utils'
 import {
@@ -74,7 +75,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
 
     const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
     if (denied) return denied
-    const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
+
+    let fileBuffer: Buffer
+    try {
+      const servable = await downloadServableFileFromStorage(userFile, requestId, logger)
+      fileBuffer = servable.buffer
+    } catch (error) {
+      const notReady = docNotReadyResponse(error)
+      if (notReady) return notReady
+      logger.error(`[${requestId}] Failed to download file from storage:`, error)
+      return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 })
+    }
+
     const resolvedFileName = data.fileName || userFile.name || 'attachment'
 
     let resolvedIP: string

apps/sim/app/api/tools/box/upload/route.ts

@@ -7,7 +7,8 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
-import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
 import { assertToolFileAccess } from '@/app/api/files/authorization'
 
 export const dynamic = 'force-dynamic'
@@ -53,7 +54,17 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
 
       const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
       if (denied) return denied
-      fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
+      try {
+        const result = await downloadServableFileFromStorage(userFile, requestId, logger)
+        fileBuffer = result.buffer
+      } catch (error) {
+        const notReady = docNotReadyResponse(error)
+        if (notReady) return notReady
+        return NextResponse.json(
+          { success: false, error: getErrorMessage(error, 'Failed to download file') },
+          { status: 500 }
+        )
+      }
       fileName = validatedData.fileName || userFile.name
     } else if (validatedData.fileContent) {
       logger.info(`[${requestId}] Using legacy base64 content input`)

apps/sim/app/api/tools/brex/upload-receipt/route.test.ts

@@ -21,7 +21,7 @@ vi.mock('@/lib/uploads/utils/file-utils', () => ({
   processFilesToUserFiles: mockProcessFilesToUserFiles,
 }))
 vi.mock('@/lib/uploads/utils/file-utils.server', () => ({
-  downloadFileFromStorage: mockDownloadFileFromStorage,
+  downloadServableFileFromStorage: mockDownloadFileFromStorage,
 }))
 vi.mock('@/app/api/files/authorization', () => ({
   assertToolFileAccess: mockAssertToolFileAccess,
@@ -65,7 +65,10 @@ beforeEach(() => {
     { key: 'uploads/receipt.pdf', name: 'receipt.pdf', size: 5, type: 'application/pdf' },
   ])
   mockAssertToolFileAccess.mockResolvedValue(null)
-  mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('receipt-bytes'))
+  mockDownloadFileFromStorage.mockResolvedValue({
+    buffer: Buffer.from('receipt-bytes'),
+    contentType: 'application/pdf',
+  })
 })
 
 describe('POST /api/tools/brex/upload-receipt', () => {
@@ -192,7 +195,10 @@ describe('POST /api/tools/brex/upload-receipt', () => {
   })
 
   it('rejects files over the 50 MB limit', async () => {
-    mockDownloadFileFromStorage.mockResolvedValueOnce(Buffer.alloc(50 * 1024 * 1024 + 1))
+    mockDownloadFileFromStorage.mockResolvedValueOnce({
+      buffer: Buffer.alloc(50 * 1024 * 1024 + 1),
+      contentType: 'application/pdf',
+    })
 
     const response = await POST(createMockRequest('POST', baseBody))
     expect(response.status).toBe(400)

apps/sim/app/api/tools/brex/upload-receipt/route.ts

@@ -11,7 +11,8 @@ import {
 import { generateRequestId } from '@/lib/core/utils/request'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
-import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
 import { assertToolFileAccess } from '@/app/api/files/authorization'
 import { BREX_API_BASE, buildBrexHeaders } from '@/tools/brex/utils'
 
@@ -48,7 +49,19 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger)
     if (denied) return denied
 
-    const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger)
+    let fileBuffer: Buffer
+    try {
+      const resolved = await downloadServableFileFromStorage(userFile, requestId, logger)
+      fileBuffer = resolved.buffer
+    } catch (error) {
+      const notReady = docNotReadyResponse(error)
+      if (notReady) return notReady
+      logger.error(`[${requestId}] Failed to download receipt file:`, error)
+      return NextResponse.json(
+        { success: false, error: getErrorMessage(error, 'Unknown error') },
+        { status: 500 }
+      )
+    }
     if (fileBuffer.length > MAX_RECEIPT_SIZE_BYTES) {
       return NextResponse.json(
         { success: false, error: 'Receipt file exceeds the 50 MB limit' },

apps/sim/app/api/tools/confluence/upload-attachment/route.ts

@@ -7,7 +7,8 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
 import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { processSingleFileToUserFile, type RawFileInput } from '@/lib/uploads/utils/file-utils'
-import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server'
+import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response'
 import { assertToolFileAccess } from '@/app/api/files/authorization'
 import { getConfluenceCloudId } from '@/tools/confluence/utils'
 import { parseAtlassianErrorMessage } from '@/tools/jira/utils'
@@ -91,9 +92,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     if (denied) return denied
 
     let fileBuffer: Buffer
+    let resolvedContentType: string
     try {
-      fileBuffer = await downloadFileFromStorage(userFile, 'confluence-upload', logger)
+      const servable = await downloadServableFileFromStorage(userFile, 'confluence-upload', logger)
+      fileBuffer = servable.buffer
+      resolvedContentType = servable.contentType
     } catch (error) {
+      const notReady = docNotReadyResponse(error)
+      if (notReady) return notReady
       logger.error('Failed to download file from storage:', error)
       return NextResponse.json(
         {
@@ -104,7 +110,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     }
 
     const uploadFileName = fileName || userFile.name || 'attachment'
-    const mimeType = userFile.type || 'application/octet-stream'
+    const mimeType = resolvedContentType || userFile.type || 'application/octet-stream'
 
     const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/${pageId}/child/attachment`