Skip to content

Commit 89a27ca

Browse files
TheodoreSpeaksTheodoreSpeaks
committed
fix(tables): restore large durable imports
1 parent 3664953 commit 89a27ca

10 files changed

Lines changed: 47 additions & 30 deletions

File tree

‎apps/docs/openapi-v2-tables.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7017,7 +7017,7 @@
70177017
"size": {
70187018
"type": "integer",
70197019
"minimum": 1,
7020-
"maximum": 26214400,
7020+
"maximum": 5368709120,
70217021
"description": "Exact CSV file size in bytes."
70227022
}
70237023
},

‎apps/sim/app/api/table/[tableId]/import/route.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import { statusForOrchestrationError } from '@/lib/core/orchestration/types'
1616
import { isMultipartError, readMultipart } from '@/lib/core/utils/multipart'
1717
import { generateRequestId } from '@/lib/core/utils/request'
1818
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
19-
import { CSV_MAX_FILE_SIZE_BYTES, type CsvHeaderMapping } from '@/lib/table'
19+
import { CSV_SYNC_MAX_FILE_SIZE_BYTES, type CsvHeaderMapping } from '@/lib/table'
2020
import { performTableCsvImport } from '@/lib/table/orchestration'
2121
import { getUserSettings } from '@/lib/users/queries'
2222
import {
@@ -53,7 +53,7 @@ export const POST = withRouteHandler(async (request: NextRequest, { params }: Ro
5353
let parsed: Awaited<ReturnType<typeof readMultipart>>
5454
try {
5555
parsed = await readMultipart(request, {
56-
maxFileBytes: CSV_MAX_FILE_SIZE_BYTES,
56+
maxFileBytes: CSV_SYNC_MAX_FILE_SIZE_BYTES,
5757
requiredFieldsBeforeFile: ['workspaceId'],
5858
signal: request.signal,
5959
})

‎apps/sim/app/api/table/import-csv/route.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import { isMultipartError, readMultipart } from '@/lib/core/utils/multipart'
1010
import { generateRequestId } from '@/lib/core/utils/request'
1111
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
1212
import { findActiveFolder } from '@/lib/folders/queries'
13-
import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table'
13+
import { CSV_SYNC_MAX_FILE_SIZE_BYTES } from '@/lib/table'
1414
import { performCreateTableFromCsv } from '@/lib/table/orchestration'
1515
import { getUserSettings } from '@/lib/users/queries'
1616
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
@@ -39,7 +39,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
3939
let parsed: Awaited<ReturnType<typeof readMultipart>>
4040
try {
4141
parsed = await readMultipart(request, {
42-
maxFileBytes: CSV_MAX_FILE_SIZE_BYTES,
42+
maxFileBytes: CSV_SYNC_MAX_FILE_SIZE_BYTES,
4343
requiredFieldsBeforeFile: ['workspaceId'],
4444
signal: request.signal,
4545
})

‎apps/sim/hooks/queries/tables.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1835,7 +1835,7 @@ export function useImportCsv() {
18351835
},
18361836
onError: (error) => {
18371837
logger.error('Failed to start CSV import:', error)
1838-
toast.error(error.message, { duration: 5000 })
1838+
toast.error(extractValidationIssues(error)[0]?.message ?? error.message, { duration: 5000 })
18391839
},
18401840
onSettled: () => {
18411841
queryClient.invalidateQueries({ queryKey: tableKeys.lists() })
@@ -1879,7 +1879,7 @@ export function useImportFileAsTable() {
18791879
},
18801880
onError: (error) => {
18811881
logger.error('Failed to start import from file:', error)
1882-
toast.error(error.message, { duration: 5000 })
1882+
toast.error(extractValidationIssues(error)[0]?.message ?? error.message, { duration: 5000 })
18831883
},
18841884
onSettled: () => {
18851885
queryClient.invalidateQueries({ queryKey: tableKeys.lists() })
@@ -1936,7 +1936,7 @@ export function useImportCsvIntoTable() {
19361936
onError: (error, variables) => {
19371937
if (handleTableLockRejection(error, queryClient, variables.tableId)) return
19381938
logger.error('Failed to start CSV import:', error)
1939-
toast.error(error.message, { duration: 5000 })
1939+
toast.error(extractValidationIssues(error)[0]?.message ?? error.message, { duration: 5000 })
19401940
},
19411941
onSettled: (_data, _error, variables) => {
19421942
invalidateRowCount(queryClient, variables.tableId)

‎apps/sim/lib/api/contracts/tables.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import {
3434
SORT_DIRECTIONS,
3535
TABLE_LIMITS,
3636
} from '@/lib/table/constants'
37-
import { CSV_MAX_FILE_SIZE_BYTES, CSV_MAX_FILE_SIZE_MESSAGE } from '@/lib/table/import'
37+
import { CSV_SYNC_MAX_FILE_SIZE_BYTES, CSV_SYNC_MAX_FILE_SIZE_MESSAGE } from '@/lib/table/import'
3838
import {
3939
getTablePredicateTreeSizeError,
4040
MAX_PREDICATE_GROUP_SIZE,
@@ -1063,10 +1063,10 @@ export const csvFileSchema = z
10631063
ctx.addIssue({ code: 'custom', message: 'CSV file is required' })
10641064
return
10651065
}
1066-
if (value.size > CSV_MAX_FILE_SIZE_BYTES) {
1066+
if (value.size > CSV_SYNC_MAX_FILE_SIZE_BYTES) {
10671067
ctx.addIssue({
10681068
code: 'custom',
1069-
message: CSV_MAX_FILE_SIZE_MESSAGE,
1069+
message: CSV_SYNC_MAX_FILE_SIZE_MESSAGE,
10701070
})
10711071
}
10721072
})

‎apps/sim/lib/api/contracts/v2/__tests__/tables.test.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import {
1111
v2UpdateTableColumnBodySchema,
1212
} from '@/lib/api/contracts/v2/tables'
1313
import { TABLE_LIMITS } from '@/lib/table/constants'
14-
import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table/import'
14+
import { CSV_DURABLE_MAX_FILE_SIZE_BYTES } from '@/lib/table/import'
1515

1616
const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8'
1717

@@ -85,10 +85,12 @@ function existingTableImport(overrides: Record<string, unknown> = {}) {
8585
describe('v2 table import contracts', () => {
8686
it('accepts the exact CSV byte limit and rejects one byte over it', () => {
8787
expect(
88-
v2TableUploadImportSourceSchema.safeParse(uploadSource(CSV_MAX_FILE_SIZE_BYTES)).success
88+
v2TableUploadImportSourceSchema.safeParse(uploadSource(CSV_DURABLE_MAX_FILE_SIZE_BYTES))
89+
.success
8990
).toBe(true)
9091
expect(
91-
v2TableUploadImportSourceSchema.safeParse(uploadSource(CSV_MAX_FILE_SIZE_BYTES + 1)).success
92+
v2TableUploadImportSourceSchema.safeParse(uploadSource(CSV_DURABLE_MAX_FILE_SIZE_BYTES + 1))
93+
.success
9294
).toBe(false)
9395
})
9496

‎apps/sim/lib/api/contracts/v2/tables.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,10 @@ import {
6262
v2UploadTransferSchema,
6363
} from '@/lib/api/contracts/v2/uploads'
6464
import { TABLE_LIMITS } from '@/lib/table/constants'
65-
import { CSV_MAX_FILE_SIZE_BYTES, CSV_MAX_FILE_SIZE_MESSAGE } from '@/lib/table/import'
65+
import {
66+
CSV_DURABLE_MAX_FILE_SIZE_BYTES,
67+
CSV_DURABLE_MAX_FILE_SIZE_MESSAGE,
68+
} from '@/lib/table/import'
6669
import type { RowData } from '@/lib/table/types'
6770

6871
/**
@@ -1405,7 +1408,7 @@ export const v2TableUploadImportSourceSchema = z
14051408
.number()
14061409
.int()
14071410
.min(1)
1408-
.max(CSV_MAX_FILE_SIZE_BYTES, CSV_MAX_FILE_SIZE_MESSAGE)
1411+
.max(CSV_DURABLE_MAX_FILE_SIZE_BYTES, CSV_DURABLE_MAX_FILE_SIZE_MESSAGE)
14091412
.describe('Exact CSV file size in bytes.'),
14101413
})
14111414
.strict()

‎apps/sim/lib/table/import.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import type { ColumnType } from '@/lib/table/column-types'
1818
import { parseCurrencyInput } from '@/lib/table/currency'
1919
import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates'
2020
import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types'
21+
import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types'
2122

2223
/**
2324
* Field separators we sniff for, in tie-break priority order. Semicolon files are
@@ -230,10 +231,15 @@ export const CSV_MAX_BATCH_SIZE = 5000
230231
/** Maximum serialized CSV row data retained before an import batch is flushed. */
231232
export const CSV_MAX_BATCH_SIZE_BYTES = 5 * 1024 * 1024
232233

233-
/** Maximum CSV/TSV file size accepted by import routes (25 MB). */
234-
export const CSV_MAX_FILE_SIZE_BYTES = 25 * 1024 * 1024
234+
/** Maximum CSV/TSV size accepted by legacy multipart routes that buffer request bodies. */
235+
export const CSV_SYNC_MAX_FILE_SIZE_BYTES = 25 * 1024 * 1024
235236

236-
export const CSV_MAX_FILE_SIZE_MESSAGE = `File exceeds maximum allowed size of ${CSV_MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB`
237+
export const CSV_SYNC_MAX_FILE_SIZE_MESSAGE = `File exceeds maximum allowed size of ${CSV_SYNC_MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB`
238+
239+
/** Maximum CSV/TSV size accepted by the bounded streaming import worker. */
240+
export const CSV_DURABLE_MAX_FILE_SIZE_BYTES = MAX_WORKSPACE_FILE_SIZE
241+
242+
export const CSV_DURABLE_MAX_FILE_SIZE_MESSAGE = 'File exceeds maximum allowed size of 5 GB'
237243

238244
/**
239245
* Error thrown when the user-supplied mapping or CSV does not line up with the

‎apps/sim/lib/table/orchestration/import-resource.test.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ vi.mock('@/lib/uploads/upload-session/service', () => ({
4646
}))
4747
vi.mock('@/lib/users/queries', () => ({ getUserSettings: mockGetUserSettings }))
4848

49-
import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table/import'
49+
import { CSV_DURABLE_MAX_FILE_SIZE_BYTES } from '@/lib/table/import'
5050
import { createAuthorizedTableImportResource } from '@/lib/table/orchestration/import-resource'
5151

5252
const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8'
@@ -108,7 +108,7 @@ describe('createAuthorizedTableImportResource workspace file size', () => {
108108
})
109109

110110
it('accepts a workspace CSV at the exact byte limit', async () => {
111-
mockGetWorkspaceFile.mockResolvedValue(workspaceFile(CSV_MAX_FILE_SIZE_BYTES))
111+
mockGetWorkspaceFile.mockResolvedValue(workspaceFile(CSV_DURABLE_MAX_FILE_SIZE_BYTES))
112112

113113
const result = await createImport({ workspaceId: WORKSPACE_ID, source: SOURCE, target: TARGET })
114114

@@ -118,7 +118,7 @@ describe('createAuthorizedTableImportResource workspace file size', () => {
118118
})
119119

120120
it('rejects a workspace CSV one byte over the limit before creating a table', async () => {
121-
mockGetWorkspaceFile.mockResolvedValue(workspaceFile(CSV_MAX_FILE_SIZE_BYTES + 1))
121+
mockGetWorkspaceFile.mockResolvedValue(workspaceFile(CSV_DURABLE_MAX_FILE_SIZE_BYTES + 1))
122122

123123
await expect(
124124
createImport({ workspaceId: WORKSPACE_ID, source: SOURCE, target: TARGET })
@@ -150,13 +150,16 @@ describe('createAuthorizedTableImportResource upload size', () => {
150150
type: 'upload',
151151
name: 'data.csv',
152152
contentType: 'text/csv',
153-
size: CSV_MAX_FILE_SIZE_BYTES,
153+
size: CSV_DURABLE_MAX_FILE_SIZE_BYTES,
154154
},
155155
target: TARGET,
156156
})
157157

158158
expect(mockCreateUploadSession).toHaveBeenCalledWith(
159-
expect.objectContaining({ fileSize: CSV_MAX_FILE_SIZE_BYTES, purpose: 'table_import' })
159+
expect.objectContaining({
160+
fileSize: CSV_DURABLE_MAX_FILE_SIZE_BYTES,
161+
purpose: 'table_import',
162+
})
160163
)
161164
})
162165

@@ -168,7 +171,7 @@ describe('createAuthorizedTableImportResource upload size', () => {
168171
type: 'upload',
169172
name: 'data.csv',
170173
contentType: 'text/csv',
171-
size: CSV_MAX_FILE_SIZE_BYTES + 1,
174+
size: CSV_DURABLE_MAX_FILE_SIZE_BYTES + 1,
172175
},
173176
target: TARGET,
174177
})

‎apps/sim/lib/table/orchestration/import-resource.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@ import { runDetached } from '@/lib/core/utils/background'
2222
import { generateRequestId } from '@/lib/core/utils/request'
2323
import { findActiveFolder } from '@/lib/folders/queries'
2424
import { getWorkspaceTableLimits } from '@/lib/table/billing'
25-
import { CSV_MAX_FILE_SIZE_BYTES, CSV_MAX_FILE_SIZE_MESSAGE } from '@/lib/table/import'
25+
import {
26+
CSV_DURABLE_MAX_FILE_SIZE_BYTES,
27+
CSV_DURABLE_MAX_FILE_SIZE_MESSAGE,
28+
} from '@/lib/table/import'
2629
import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner'
2730
import { markTableJobRunningInWorkspace } from '@/lib/table/jobs/service'
2831
import { assertRowDelete, assertRowInsert } from '@/lib/table/mutation-locks'
@@ -85,8 +88,8 @@ async function createTableImportResourceCore(
8588

8689
if (body.source.type === 'upload') {
8790
assertCsvFileName(body.source.name)
88-
if (body.source.size > CSV_MAX_FILE_SIZE_BYTES) {
89-
throw new OrchestrationError('validation', CSV_MAX_FILE_SIZE_MESSAGE)
91+
if (body.source.size > CSV_DURABLE_MAX_FILE_SIZE_BYTES) {
92+
throw new OrchestrationError('validation', CSV_DURABLE_MAX_FILE_SIZE_MESSAGE)
9093
}
9194
const upload = await createUploadSession({
9295
id: importId,
@@ -523,8 +526,8 @@ async function requireWorkspaceSource(
523526
if (!resolved || resolved.id !== fileId || resolved.workspaceId !== workspaceId) {
524527
throw new OrchestrationError('not_found', 'Workspace file not found')
525528
}
526-
if (resolved.size > CSV_MAX_FILE_SIZE_BYTES) {
527-
throw new OrchestrationError('validation', CSV_MAX_FILE_SIZE_MESSAGE)
529+
if (resolved.size > CSV_DURABLE_MAX_FILE_SIZE_BYTES) {
530+
throw new OrchestrationError('validation', CSV_DURABLE_MAX_FILE_SIZE_MESSAGE)
528531
}
529532
return resolved
530533
}

0 commit comments

Comments
 (0)