From cedd01bac66d67f386bfb16521b16c882375ad7d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:42:09 +0000 Subject: [PATCH 1/9] Add Excel (.xlsx) export endpoint for API datasets Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- README.md | 33 ++++++ src/export/datasets.js | 224 ++++++++++++++++++++++++++++++++++++++++ src/export/router.js | 57 ++++++++++ src/export/xlsx.js | 229 +++++++++++++++++++++++++++++++++++++++++ src/index.js | 4 + test/export.test.js | 196 +++++++++++++++++++++++++++++++++++ 6 files changed, 743 insertions(+) create mode 100644 src/export/datasets.js create mode 100644 src/export/router.js create mode 100644 src/export/xlsx.js create mode 100644 test/export.test.js diff --git a/README.md b/README.md index 702f776..c751773 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,11 @@ src/ connectors/ # Replaceable OpenTrading, Portfolio-Watcher, tax-break adapters data/store.js # In-memory data store with seed data domain/finance.js # Canonical finance models and normalization helpers + export/ # Excel (.xlsx) writer, dataset registry, and /export routes services/financeService.js # Finance aggregation, caching, and error handling test/ graphql.test.js # API tests executed against the schema + export.test.js # Excel export writer, dataset, and route tests website/ src/App.jsx # Learning site: primer, tips, API Explorer src/backendSamples.js # GraphQL server samples in 10 backend languages @@ -60,6 +62,7 @@ environment variable): - Liveness check: - Readiness check (per-upstream): - Metrics (Prometheus text): +- Excel exports: Opening the GraphQL endpoint in a browser loads the Apollo Sandbox, where you can explore the schema and run the operations below. @@ -398,6 +401,36 @@ curl http://localhost:4000/graphql \ -d '{"query":"{ users { id name posts { title } } }"}' ``` +## Excel export + +Any dataset the API serves can be downloaded as an Excel workbook (`.xlsx`) — +useful for sharing a portfolio snapshot or trade history with a spreadsheet. +The files are generated in-process, so no extra dependency or service is needed. + +```bash +curl http://localhost:4000/export # list datasets +curl -O -J http://localhost:4000/export/trades.xlsx # download a workbook +``` + +| Dataset | Path | Sheets | +| --- | --- | --- | +| `users` | `/export/users.xlsx` | Users (with post counts) | +| `posts` | `/export/posts.xlsx` | Posts (with author names) | +| `portfolio` | `/export/portfolio.xlsx` | Accounts, Positions, Performance | +| `trades` | `/export/trades.xlsx` | Trades, Orders, Tax Events | +| `tax-estimate` | `/export/tax-estimate.xlsx?taxYear=2024` | Summary, Tax Events | + +Finance exports accept the same query parameters as their GraphQL counterparts +(`accountId`, `symbol`, `side`, `status`, `from`, `to`, `limit`, `offset`, and +`taxYear`, which is required for `tax-estimate`): + +```bash +curl -O -J 'http://localhost:4000/export/trades.xlsx?accountId=acct-1&symbol=AAPL&limit=50' +``` + +When upstream connectors return partial data, the workbook gains an extra +`Errors` sheet describing each failure instead of hiding the gap. + ## Notes Data lives in memory only, so every restart resets the service to its seed data diff --git a/src/export/datasets.js b/src/export/datasets.js new file mode 100644 index 0000000..9cd6343 --- /dev/null +++ b/src/export/datasets.js @@ -0,0 +1,224 @@ +/** + * Exportable datasets. + * + * Every dataset describes the sheets of a workbook plus a loader that pulls the + * rows from the same store/finance service the GraphQL resolvers use, so an + * export always matches what the API returns. + */ + +/** Parses the shared pagination/filter query parameters of an export request. */ +function readParams(params = {}) { + const optionalInt = (value) => { + if (value === undefined || value === null || value === '') return undefined; + const parsed = Number.parseInt(value, 10); + return Number.isNaN(parsed) ? undefined : parsed; + }; + const optionalString = (value) => (typeof value === 'string' && value !== '' ? value : undefined); + + return { + accountId: optionalString(params.accountId), + symbol: optionalString(params.symbol), + side: optionalString(params.side), + status: optionalString(params.status), + from: optionalString(params.from), + to: optionalString(params.to), + limit: optionalInt(params.limit), + offset: optionalInt(params.offset), + taxYear: optionalInt(params.taxYear), + }; +} + +const USER_COLUMNS = [ + { key: 'id', header: 'Id' }, + { key: 'name', header: 'Name' }, + { key: 'email', header: 'Email' }, + { key: 'postCount', header: 'Posts' }, +]; + +const POST_COLUMNS = [ + { key: 'id', header: 'Id' }, + { key: 'title', header: 'Title' }, + { key: 'content', header: 'Content' }, + { key: 'authorId', header: 'Author Id' }, + { key: 'authorName', header: 'Author' }, +]; + +const ACCOUNT_COLUMNS = [ + { key: 'id', header: 'Id' }, + { key: 'name', header: 'Name' }, + { key: 'type', header: 'Type' }, + { key: 'currency', header: 'Currency' }, + { key: 'provider', header: 'Provider' }, +]; + +const POSITION_COLUMNS = [ + { key: 'id', header: 'Id' }, + { key: 'accountId', header: 'Account Id' }, + { key: 'symbol', header: 'Symbol' }, + { key: 'quantity', header: 'Quantity' }, + { key: 'averageCost', header: 'Average Cost' }, + { key: 'marketPrice', header: 'Market Price' }, + { key: 'marketValue', header: 'Market Value' }, + { key: 'unrealizedPnL', header: 'Unrealized P/L' }, +]; + +const PERFORMANCE_COLUMNS = [ + { key: 'id', header: 'Id' }, + { key: 'accountId', header: 'Account Id' }, + { key: 'asOf', header: 'As Of' }, + { key: 'totalValue', header: 'Total Value' }, + { key: 'cash', header: 'Cash' }, + { key: 'marketValue', header: 'Market Value' }, + { key: 'dayPnL', header: 'Day P/L' }, + { key: 'totalPnL', header: 'Total P/L' }, +]; + +const TRADE_COLUMNS = [ + { key: 'id', header: 'Id' }, + { key: 'accountId', header: 'Account Id' }, + { key: 'orderId', header: 'Order Id' }, + { key: 'symbol', header: 'Symbol' }, + { key: 'side', header: 'Side' }, + { key: 'quantity', header: 'Quantity' }, + { key: 'price', header: 'Price' }, + { key: 'status', header: 'Status' }, + { key: 'executedAt', header: 'Executed At' }, +]; + +const ORDER_COLUMNS = [ + { key: 'id', header: 'Id' }, + { key: 'accountId', header: 'Account Id' }, + { key: 'symbol', header: 'Symbol' }, + { key: 'side', header: 'Side' }, + { key: 'quantity', header: 'Quantity' }, + { key: 'limitPrice', header: 'Limit Price' }, + { key: 'status', header: 'Status' }, + { key: 'createdAt', header: 'Created At' }, +]; + +const TAX_EVENT_COLUMNS = [ + { key: 'id', header: 'Id' }, + { key: 'tradeId', header: 'Trade Id' }, + { key: 'symbol', header: 'Symbol' }, + { key: 'quantity', header: 'Quantity' }, + { key: 'proceeds', header: 'Proceeds' }, + { key: 'costBasis', header: 'Cost Basis' }, + { key: 'realizedGain', header: 'Realized Gain' }, + { key: 'holdingPeriod', header: 'Holding Period' }, + { key: 'occurredAt', header: 'Occurred At' }, +]; + +/** Registry of everything the /export endpoint can produce. */ +export const datasets = { + users: { + filename: 'users', + async load(_params, { store }) { + const users = store.listUsers(); + return { + sheets: [ + { + name: 'Users', + columns: USER_COLUMNS, + rows: users.map((user) => ({ ...user, postCount: store.listPostsByAuthor(user.id).length })), + }, + ], + }; + }, + }, + + posts: { + filename: 'posts', + async load(_params, { store }) { + return { + sheets: [ + { + name: 'Posts', + columns: POST_COLUMNS, + rows: store.listPosts().map((post) => ({ + ...post, + authorName: store.getUser(post.authorId)?.name ?? '', + })), + }, + ], + }; + }, + }, + + portfolio: { + filename: 'portfolio-overview', + async load(params, { finance }) { + const overview = await finance.portfolioOverview(params); + return { + sheets: [ + { name: 'Accounts', columns: ACCOUNT_COLUMNS, rows: overview.accounts }, + { name: 'Positions', columns: POSITION_COLUMNS, rows: overview.positions }, + { name: 'Performance', columns: PERFORMANCE_COLUMNS, rows: overview.performance }, + ], + errors: overview.errors, + }; + }, + }, + + trades: { + filename: 'trade-history', + async load(params, { finance }) { + const history = await finance.tradeHistory(params); + return { + sheets: [ + { name: 'Trades', columns: TRADE_COLUMNS, rows: history.trades }, + { name: 'Orders', columns: ORDER_COLUMNS, rows: history.orders }, + { name: 'Tax Events', columns: TAX_EVENT_COLUMNS, rows: history.taxEvents }, + ], + errors: history.errors, + }; + }, + }, + + 'tax-estimate': { + filename: 'tax-estimate', + async load(params, { finance }) { + if (params.taxYear === undefined) { + throw Object.assign(new Error('taxYear query parameter is required'), { statusCode: 400 }); + } + + const summary = await finance.taxEstimate(params); + return { + sheets: [ + { + name: 'Summary', + columns: [ + { key: 'taxYear', header: 'Tax Year' }, + { key: 'currency', header: 'Currency' }, + { key: 'totalProceeds', header: 'Total Proceeds' }, + { key: 'totalCostBasis', header: 'Total Cost Basis' }, + { key: 'realizedGain', header: 'Realized Gain' }, + { key: 'estimatedTax', header: 'Estimated Tax' }, + { key: 'taxRate', header: 'Tax Rate' }, + ], + rows: [summary], + }, + { name: 'Tax Events', columns: TAX_EVENT_COLUMNS, rows: summary.events }, + ], + errors: summary.errors, + }; + }, + }, +}; + +/** Dataset names accepted by the export endpoint. */ +export const datasetNames = Object.keys(datasets); + +/** + * Loads a dataset into workbook sheets. + * + * @throws {Error & {statusCode: number}} when the dataset is unknown or params are invalid + */ +export async function loadDataset(name, params, context) { + const dataset = datasets[name]; + if (!dataset) { + throw Object.assign(new Error(`unknown dataset "${name}"`), { statusCode: 404, datasets: datasetNames }); + } + + const result = await dataset.load(readParams(params), context); + return { filename: dataset.filename, ...result }; +} diff --git a/src/export/router.js b/src/export/router.js new file mode 100644 index 0000000..fb00844 --- /dev/null +++ b/src/export/router.js @@ -0,0 +1,57 @@ +import express from 'express'; + +import { loadDataset, datasetNames } from './datasets.js'; +import { buildWorkbook, XLSX_CONTENT_TYPE } from './xlsx.js'; + +const ERROR_COLUMNS = [ + { key: 'source', header: 'Source' }, + { key: 'code', header: 'Code' }, + { key: 'category', header: 'Category' }, + { key: 'status', header: 'Status' }, + { key: 'retryable', header: 'Retryable' }, + { key: 'message', header: 'Message' }, +]; + +/** + * Express router exposing spreadsheet downloads. + * + * `GET /export` lists the datasets, `GET /export/:dataset(.xlsx)` streams the + * workbook. Finance datasets accept the same filter/pagination query + * parameters as their GraphQL counterparts. + */ +export function createExportRouter({ store, finance, logger } = {}) { + const router = express.Router(); + + router.get('/', (_req, res) => { + res.json({ datasets: datasetNames, format: 'xlsx' }); + }); + + router.get('/:dataset', async (req, res) => { + // Accept both /export/users and /export/users.xlsx. + const name = req.params.dataset.replace(/\.xlsx$/i, ''); + + try { + const { filename, sheets, errors = [] } = await loadDataset(name, req.query, { store, finance }); + // Partial finance results carry upstream errors; surface them in the + // workbook instead of silently shipping incomplete data. + const allSheets = errors.length > 0 ? [...sheets, { name: 'Errors', columns: ERROR_COLUMNS, rows: errors }] : sheets; + const workbook = buildWorkbook({ sheets: allSheets }); + + res.set('content-type', XLSX_CONTENT_TYPE); + res.set('content-disposition', `attachment; filename="${filename}.xlsx"`); + res.set('content-length', String(workbook.length)); + res.send(workbook); + } catch (error) { + const statusCode = error?.statusCode ?? 500; + if (statusCode >= 500) { + logger?.error('export failed', { dataset: name, error: error?.message ?? String(error) }); + } + res.status(statusCode).json({ + error: statusCode >= 500 ? 'export failed' : error.message, + ...(error?.datasets ? { datasets: error.datasets } : {}), + }); + } + }); + + return router; +} diff --git a/src/export/xlsx.js b/src/export/xlsx.js new file mode 100644 index 0000000..2afc5cc --- /dev/null +++ b/src/export/xlsx.js @@ -0,0 +1,229 @@ +/** + * Minimal, dependency-free writer for the Office Open XML (.xlsx) format. + * + * An .xlsx file is a ZIP archive holding a handful of XML parts. Only the + * parts required for plain data sheets are emitted here, which keeps the + * service free of a heavyweight spreadsheet dependency while still producing + * a file Excel, Numbers, and LibreOffice open natively. + */ +import { deflateRawSync } from 'node:zlib'; + +const CRC_TABLE = (() => { + const table = new Uint32Array(256); + for (let i = 0; i < 256; i += 1) { + let value = i; + for (let bit = 0; bit < 8; bit += 1) { + value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1; + } + table[i] = value >>> 0; + } + return table; +})(); + +function crc32(buffer) { + let crc = 0xffffffff; + for (const byte of buffer) crc = CRC_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8); + return (crc ^ 0xffffffff) >>> 0; +} + +function escapeXml(value) { + return ( + String(value) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') + // Control characters are illegal in XML 1.0 and would corrupt the file. + .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/g, '') + ); +} + +/** Converts a zero-based column index into its spreadsheet letters (0 -> A). */ +export function columnName(index) { + let name = ''; + let remaining = index; + do { + name = String.fromCharCode(65 + (remaining % 26)) + name; + remaining = Math.floor(remaining / 26) - 1; + } while (remaining >= 0); + return name; +} + +/** Excel sheet names cannot exceed 31 chars or contain : \ / ? * [ ]. */ +function sanitizeSheetName(name, fallback) { + const cleaned = String(name ?? '') + .replace(/[:\\/?*[\]]/g, ' ') + .trim(); + return (cleaned || fallback).slice(0, 31); +} + +function renderCell(reference, value) { + if (value === null || value === undefined || value === '') return ``; + if (typeof value === 'number' && Number.isFinite(value)) return `${value}`; + if (typeof value === 'boolean') return `${value ? 1 : 0}`; + return `${escapeXml(value)}`; +} + +function renderRow(values, rowNumber) { + const cells = values.map((value, index) => renderCell(`${columnName(index)}${rowNumber}`, value)).join(''); + return `${cells}`; +} + +function renderSheet({ columns, rows }) { + const header = renderRow(columns.map((column) => column.header ?? column.key), 1); + const body = rows + .map((row, index) => renderRow(columns.map((column) => row[column.key] ?? null), index + 2)) + .join(''); + + return ( + `` + + `` + + `${header}${body}` + ); +} + +function zipEntry(name, contents, offset) { + const nameBuffer = Buffer.from(name, 'utf8'); + const data = Buffer.from(contents, 'utf8'); + const compressed = deflateRawSync(data); + const crc = crc32(data); + + const localHeader = Buffer.alloc(30); + localHeader.writeUInt32LE(0x04034b50, 0); + localHeader.writeUInt16LE(20, 4); // version needed to extract + localHeader.writeUInt16LE(0, 6); // flags + localHeader.writeUInt16LE(8, 8); // deflate + localHeader.writeUInt16LE(0, 10); // modification time + localHeader.writeUInt16LE(0x21, 12); // modification date (1980-01-01) + localHeader.writeUInt32LE(crc, 14); + localHeader.writeUInt32LE(compressed.length, 18); + localHeader.writeUInt32LE(data.length, 22); + localHeader.writeUInt16LE(nameBuffer.length, 26); + localHeader.writeUInt16LE(0, 28); // extra field length + + const centralHeader = Buffer.alloc(46); + centralHeader.writeUInt32LE(0x02014b50, 0); + centralHeader.writeUInt16LE(20, 4); // version made by + centralHeader.writeUInt16LE(20, 6); // version needed to extract + centralHeader.writeUInt16LE(0, 8); + centralHeader.writeUInt16LE(8, 10); + centralHeader.writeUInt16LE(0, 12); + centralHeader.writeUInt16LE(0x21, 14); + centralHeader.writeUInt32LE(crc, 16); + centralHeader.writeUInt32LE(compressed.length, 20); + centralHeader.writeUInt32LE(data.length, 24); + centralHeader.writeUInt16LE(nameBuffer.length, 28); + centralHeader.writeUInt16LE(0, 30); // extra field length + centralHeader.writeUInt16LE(0, 32); // comment length + centralHeader.writeUInt16LE(0, 34); // disk number + centralHeader.writeUInt16LE(0, 36); // internal attributes + centralHeader.writeUInt32LE(0, 38); // external attributes + centralHeader.writeUInt32LE(offset, 42); + + return { + local: Buffer.concat([localHeader, nameBuffer, compressed]), + central: Buffer.concat([centralHeader, nameBuffer]), + }; +} + +function zip(files) { + const locals = []; + const centrals = []; + let offset = 0; + + for (const [name, contents] of files) { + const entry = zipEntry(name, contents, offset); + locals.push(entry.local); + centrals.push(entry.central); + offset += entry.local.length; + } + + const centralDirectory = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(0, 4); // disk number + end.writeUInt16LE(0, 6); // central directory start disk + end.writeUInt16LE(files.length, 8); + end.writeUInt16LE(files.length, 10); + end.writeUInt32LE(centralDirectory.length, 12); + end.writeUInt32LE(offset, 16); + end.writeUInt16LE(0, 20); // comment length + + return Buffer.concat([...locals, centralDirectory, end]); +} + +/** + * Builds an .xlsx workbook buffer. + * + * @param {{sheets: Array<{name?: string, columns: Array<{key: string, header?: string}>, rows?: object[]}>}} workbook + * @returns {Buffer} the binary .xlsx document + */ +export function buildWorkbook({ sheets = [] } = {}) { + if (sheets.length === 0) throw new TypeError('a workbook needs at least one sheet'); + + const normalized = sheets.map((sheet, index) => { + if (!Array.isArray(sheet.columns) || sheet.columns.length === 0) { + throw new TypeError('each sheet needs at least one column'); + } + + return { + name: sanitizeSheetName(sheet.name, `Sheet${index + 1}`), + columns: sheet.columns, + rows: sheet.rows ?? [], + }; + }); + + const sheetEntries = normalized.map((sheet, index) => [`xl/worksheets/sheet${index + 1}.xml`, renderSheet(sheet)]); + + const contentTypes = + `` + + `` + + `` + + `` + + `` + + normalized + .map( + (_sheet, index) => + `` + ) + .join('') + + ``; + + const rootRels = + `` + + `` + + `` + + ``; + + const workbookXml = + `` + + `` + + normalized + .map((sheet, index) => ``) + .join('') + + ``; + + const workbookRels = + `` + + `` + + normalized + .map( + (_sheet, index) => + `` + ) + .join('') + + ``; + + return zip([ + ['[Content_Types].xml', contentTypes], + ['_rels/.rels', rootRels], + ['xl/workbook.xml', workbookXml], + ['xl/_rels/workbook.xml.rels', workbookRels], + ...sheetEntries, + ]); +} + +/** MIME type browsers and Excel associate with .xlsx downloads. */ +export const XLSX_CONTENT_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; diff --git a/src/index.js b/src/index.js index d6ba649..a5d83c7 100644 --- a/src/index.js +++ b/src/index.js @@ -3,6 +3,7 @@ import cors from 'cors'; import express from 'express'; import { store } from './data/store.js'; +import { createExportRouter } from './export/router.js'; import { logger } from './observability/logger.js'; import { metrics } from './observability/metrics.js'; import { financeService } from './services/financeService.js'; @@ -32,6 +33,9 @@ async function main() { res.set('content-type', 'text/plain; version=0.0.4').send(metrics.toPrometheus()); }); + // Spreadsheet downloads (.xlsx) for the same data the GraphQL API serves. + app.use('/export', cors(), createExportRouter({ store, finance: financeService, logger })); + app.use( '/graphql', cors(), diff --git a/test/export.test.js b/test/export.test.js new file mode 100644 index 0000000..0509444 --- /dev/null +++ b/test/export.test.js @@ -0,0 +1,196 @@ +import assert from 'node:assert/strict'; +import { beforeEach, describe, it } from 'node:test'; +import { inflateRawSync } from 'node:zlib'; + +import { createStore } from '../src/data/store.js'; +import { loadDataset, datasetNames } from '../src/export/datasets.js'; +import { createExportRouter } from '../src/export/router.js'; +import { buildWorkbook, columnName, XLSX_CONTENT_TYPE } from '../src/export/xlsx.js'; +import { createFinanceService } from '../src/services/financeService.js'; + +/** Reads the entries of a ZIP archive produced by the workbook writer. */ +function readZip(buffer) { + const entries = new Map(); + let offset = 0; + + while (offset + 4 <= buffer.length && buffer.readUInt32LE(offset) === 0x04034b50) { + const compressedSize = buffer.readUInt32LE(offset + 18); + const nameLength = buffer.readUInt16LE(offset + 26); + const extraLength = buffer.readUInt16LE(offset + 28); + const name = buffer.toString('utf8', offset + 30, offset + 30 + nameLength); + const dataStart = offset + 30 + nameLength + extraLength; + entries.set(name, inflateRawSync(buffer.subarray(dataStart, dataStart + compressedSize)).toString('utf8')); + offset = dataStart + compressedSize; + } + + return entries; +} + +/** Minimal express-like harness so the router can be exercised without a port. */ +async function callRouter(router, { path, query = {} }) { + const headers = {}; + const response = { + statusCode: 200, + body: undefined, + set(name, value) { + headers[name] = value; + return this; + }, + status(code) { + this.statusCode = code; + return this; + }, + json(payload) { + this.body = payload; + this.finish(); + return this; + }, + send(payload) { + this.body = payload; + this.finish(); + return this; + }, + }; + + return new Promise((resolve, reject) => { + response.finish = () => resolve({ status: response.statusCode, headers, body: response.body }); + router.handle({ method: 'GET', url: path, query, params: {} }, response, (error) => + error ? reject(error) : resolve({ status: 404, headers, body: undefined }) + ); + }); +} + +describe('xlsx writer', () => { + it('names columns beyond Z', () => { + assert.equal(columnName(0), 'A'); + assert.equal(columnName(25), 'Z'); + assert.equal(columnName(26), 'AA'); + assert.equal(columnName(27), 'AB'); + }); + + it('produces a readable workbook with typed cells', () => { + const buffer = buildWorkbook({ + sheets: [ + { + name: 'Sheet [1]: data', + columns: [ + { key: 'symbol', header: 'Symbol' }, + { key: 'quantity', header: 'Quantity' }, + { key: 'note', header: 'Note' }, + ], + rows: [{ symbol: 'ACME', quantity: 12.5, note: 'buy & hold ' }], + }, + ], + }); + + assert.equal(buffer.readUInt32LE(0), 0x04034b50); + + const entries = readZip(buffer); + assert.ok(entries.has('[Content_Types].xml')); + assert.ok(entries.has('xl/workbook.xml')); + + const sheet = entries.get('xl/worksheets/sheet1.xml'); + assert.match(sheet, /Symbol<\/t>/); + assert.match(sheet, /12.5<\/v><\/c>/); + assert.match(sheet, /buy & hold <fast>/); + // Illegal sheet-name characters are stripped. + assert.match(entries.get('xl/workbook.xml'), /name="Sheet 1 data"/); + }); + + it('writes empty cells for missing values', () => { + const entries = readZip( + buildWorkbook({ sheets: [{ columns: [{ key: 'a' }, { key: 'b' }], rows: [{ a: 'x' }] }] }) + ); + + assert.match(entries.get('xl/worksheets/sheet1.xml'), //); + }); + + it('rejects workbooks without sheets or columns', () => { + assert.throws(() => buildWorkbook({ sheets: [] }), TypeError); + assert.throws(() => buildWorkbook({ sheets: [{ columns: [] }] }), TypeError); + }); +}); + +describe('export datasets', () => { + let store; + let finance; + + beforeEach(() => { + store = createStore(); + finance = createFinanceService(); + }); + + it('exposes the expected dataset names', () => { + assert.deepEqual(datasetNames, ['users', 'posts', 'portfolio', 'trades', 'tax-estimate']); + }); + + it('exports users with their post counts', async () => { + const { sheets } = await loadDataset('users', {}, { store, finance }); + + assert.equal(sheets.length, 1); + assert.equal(sheets[0].rows.length, 2); + assert.equal(sheets[0].rows[0].postCount, 1); + }); + + it('exports posts with the resolved author name', async () => { + const { sheets } = await loadDataset('posts', {}, { store, finance }); + + assert.equal(sheets[0].rows[0].authorName, 'Ada Lovelace'); + }); + + it('exports portfolio, trade, and tax data', async () => { + const portfolio = await loadDataset('portfolio', {}, { store, finance }); + const trades = await loadDataset('trades', { limit: '1' }, { store, finance }); + const tax = await loadDataset('tax-estimate', { taxYear: '2024' }, { store, finance }); + + assert.deepEqual(portfolio.sheets.map((sheet) => sheet.name), ['Accounts', 'Positions', 'Performance']); + assert.equal(trades.sheets[0].rows.length, 1); + assert.deepEqual(tax.sheets.map((sheet) => sheet.name), ['Summary', 'Tax Events']); + assert.equal(tax.sheets[0].rows[0].taxYear, 2024); + }); + + it('requires a tax year for the tax estimate export', async () => { + await assert.rejects(() => loadDataset('tax-estimate', {}, { store, finance }), /taxYear/); + }); + + it('rejects unknown datasets', async () => { + await assert.rejects(() => loadDataset('nope', {}, { store, finance }), (error) => { + assert.equal(error.statusCode, 404); + return true; + }); + }); +}); + +describe('export router', () => { + let router; + + beforeEach(() => { + router = createExportRouter({ store: createStore(), finance: createFinanceService() }); + }); + + it('lists available datasets', async () => { + const response = await callRouter(router, { path: '/' }); + + assert.equal(response.status, 200); + assert.deepEqual(response.body, { datasets: datasetNames, format: 'xlsx' }); + }); + + it('downloads a workbook for a dataset', async () => { + const response = await callRouter(router, { path: '/users.xlsx' }); + + assert.equal(response.status, 200); + assert.equal(response.headers['content-type'], XLSX_CONTENT_TYPE); + assert.equal(response.headers['content-disposition'], 'attachment; filename="users.xlsx"'); + assert.match(readZip(response.body).get('xl/worksheets/sheet1.xml'), /Ada Lovelace/); + }); + + it('returns 404 for unknown datasets and 400 for missing parameters', async () => { + const unknown = await callRouter(router, { path: '/unknown' }); + const missingYear = await callRouter(router, { path: '/tax-estimate' }); + + assert.equal(unknown.status, 404); + assert.deepEqual(unknown.body.datasets, datasetNames); + assert.equal(missingYear.status, 400); + assert.match(missingYear.body.error, /taxYear/); + }); +}); From 3843922e176fab76214632fda60a57ab4c0cda9f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:22:14 +0000 Subject: [PATCH 2/9] Remove XML-forbidden XLSX noncharacters Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- src/export/xlsx.js | 2 +- test/export.test.js | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/export/xlsx.js b/src/export/xlsx.js index 2afc5cc..e4b2964 100644 --- a/src/export/xlsx.js +++ b/src/export/xlsx.js @@ -35,7 +35,7 @@ function escapeXml(value) { .replace(/"/g, '"') .replace(/'/g, ''') // Control characters are illegal in XML 1.0 and would corrupt the file. - .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/g, '') + .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\ufffe\uffff]/g, '') ); } diff --git a/test/export.test.js b/test/export.test.js index 0509444..7eb0eb7 100644 --- a/test/export.test.js +++ b/test/export.test.js @@ -105,6 +105,16 @@ describe('xlsx writer', () => { assert.match(entries.get('xl/worksheets/sheet1.xml'), //); }); + it('removes XML-forbidden noncharacters from strings', () => { + const entries = readZip( + buildWorkbook({ sheets: [{ columns: [{ key: 'a' }], rows: [{ a: 'bad\ufffegap\uffffend' }] }] }) + ); + + const sheet = entries.get('xl/worksheets/sheet1.xml'); + assert.match(sheet, /badgapend/); + assert.doesNotMatch(sheet, /\ufffe|\uffff/); + }); + it('rejects workbooks without sheets or columns', () => { assert.throws(() => buildWorkbook({ sheets: [] }), TypeError); assert.throws(() => buildWorkbook({ sheets: [{ columns: [] }] }), TypeError); From 61b06323a1a5406bdf5af6b59b5cd3cbebef5e23 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:23:03 +0000 Subject: [PATCH 3/9] Clarify XLSX XML sanitizer comment Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- src/export/xlsx.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/export/xlsx.js b/src/export/xlsx.js index e4b2964..33c7867 100644 --- a/src/export/xlsx.js +++ b/src/export/xlsx.js @@ -34,7 +34,7 @@ function escapeXml(value) { .replace(/>/g, '>') .replace(/"/g, '"') .replace(/'/g, ''') - // Control characters are illegal in XML 1.0 and would corrupt the file. + // XML-forbidden control characters and noncharacters would corrupt the file. .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\ufffe\uffff]/g, '') ); } From 6b0a808e9d55a1704c38cfaa1b5fdbb1c033accb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:26:07 +0000 Subject: [PATCH 4/9] Validate export params and bound row counts Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- src/export/datasets.js | 70 ++++++++++++++++++++++++++--------- src/export/router.js | 11 +++++- src/resolvers.js | 20 +--------- src/validation/financeArgs.js | 32 ++++++++++++++++ test/export.test.js | 34 +++++++++++++++++ 5 files changed, 129 insertions(+), 38 deletions(-) create mode 100644 src/validation/financeArgs.js diff --git a/src/export/datasets.js b/src/export/datasets.js index 9cd6343..6a961d9 100644 --- a/src/export/datasets.js +++ b/src/export/datasets.js @@ -5,27 +5,57 @@ * rows from the same store/finance service the GraphQL resolvers use, so an * export always matches what the API returns. */ +import { isGraphQLInt, validateFinanceArgs } from '../validation/financeArgs.js'; /** Parses the shared pagination/filter query parameters of an export request. */ -function readParams(params = {}) { - const optionalInt = (value) => { - if (value === undefined || value === null || value === '') return undefined; - const parsed = Number.parseInt(value, 10); - return Number.isNaN(parsed) ? undefined : parsed; +function readParams(params = {}, options = {}) { + const issues = []; + const optionalInt = (name) => { + if (!Object.hasOwn(params, name) || params[name] === undefined || params[name] === null) return undefined; + const raw = params[name]; + if (Array.isArray(raw) || raw === '') { + issues.push(`${name} must be an integer`); + return undefined; + } + const value = String(raw); + if (!/^-?\d+$/.test(value)) { + issues.push(`${name} must be an integer`); + return undefined; + } + const parsed = Number(value); + if (!isGraphQLInt(parsed)) { + issues.push(`${name} must be a 32-bit integer`); + return undefined; + } + return parsed; }; - const optionalString = (value) => (typeof value === 'string' && value !== '' ? value : undefined); - - return { - accountId: optionalString(params.accountId), - symbol: optionalString(params.symbol), - side: optionalString(params.side), - status: optionalString(params.status), - from: optionalString(params.from), - to: optionalString(params.to), - limit: optionalInt(params.limit), - offset: optionalInt(params.offset), - taxYear: optionalInt(params.taxYear), + const optionalString = (name) => { + if (!Object.hasOwn(params, name) || params[name] === undefined || params[name] === null) return undefined; + if (Array.isArray(params[name])) { + issues.push(`${name} must be a string`); + return undefined; + } + return String(params[name]); }; + + const parsed = { + accountId: optionalString('accountId'), + symbol: optionalString('symbol'), + side: optionalString('side'), + status: optionalString('status'), + from: optionalString('from'), + to: optionalString('to'), + limit: optionalInt('limit'), + offset: optionalInt('offset'), + taxYear: optionalInt('taxYear'), + }; + + issues.push(...validateFinanceArgs(parsed, options)); + if (issues.length > 0) { + throw Object.assign(new Error(`invalid export query parameters: ${issues.join('; ')}`), { statusCode: 400 }); + } + + return parsed; } const USER_COLUMNS = [ @@ -146,6 +176,7 @@ export const datasets = { portfolio: { filename: 'portfolio-overview', + financeArgs: true, async load(params, { finance }) { const overview = await finance.portfolioOverview(params); return { @@ -161,6 +192,7 @@ export const datasets = { trades: { filename: 'trade-history', + financeArgs: true, async load(params, { finance }) { const history = await finance.tradeHistory(params); return { @@ -176,6 +208,7 @@ export const datasets = { 'tax-estimate': { filename: 'tax-estimate', + financeArgs: { requireTaxYear: true }, async load(params, { finance }) { if (params.taxYear === undefined) { throw Object.assign(new Error('taxYear query parameter is required'), { statusCode: 400 }); @@ -219,6 +252,7 @@ export async function loadDataset(name, params, context) { throw Object.assign(new Error(`unknown dataset "${name}"`), { statusCode: 404, datasets: datasetNames }); } - const result = await dataset.load(readParams(params), context); + const parsedParams = dataset.financeArgs ? readParams(params, dataset.financeArgs === true ? {} : dataset.financeArgs) : {}; + const result = await dataset.load(parsedParams, context); return { filename: dataset.filename, ...result }; } diff --git a/src/export/router.js b/src/export/router.js index fb00844..8a4e78c 100644 --- a/src/export/router.js +++ b/src/export/router.js @@ -11,6 +11,11 @@ const ERROR_COLUMNS = [ { key: 'retryable', header: 'Retryable' }, { key: 'message', header: 'Message' }, ]; +const DEFAULT_MAX_EXPORT_ROWS = 10000; + +function countRows(sheets) { + return sheets.reduce((total, sheet) => total + (sheet.rows?.length ?? 0), 0); +} /** * Express router exposing spreadsheet downloads. @@ -19,7 +24,7 @@ const ERROR_COLUMNS = [ * workbook. Finance datasets accept the same filter/pagination query * parameters as their GraphQL counterparts. */ -export function createExportRouter({ store, finance, logger } = {}) { +export function createExportRouter({ store, finance, logger, maxExportRows = DEFAULT_MAX_EXPORT_ROWS } = {}) { const router = express.Router(); router.get('/', (_req, res) => { @@ -35,6 +40,10 @@ export function createExportRouter({ store, finance, logger } = {}) { // Partial finance results carry upstream errors; surface them in the // workbook instead of silently shipping incomplete data. const allSheets = errors.length > 0 ? [...sheets, { name: 'Errors', columns: ERROR_COLUMNS, rows: errors }] : sheets; + const rowCount = countRows(allSheets); + if (rowCount > maxExportRows) { + throw Object.assign(new Error(`export contains ${rowCount} rows, which exceeds the limit of ${maxExportRows}`), { statusCode: 413 }); + } const workbook = buildWorkbook({ sheets: allSheets }); res.set('content-type', XLSX_CONTENT_TYPE); diff --git a/src/resolvers.js b/src/resolvers.js index 2c80259..a95bd85 100644 --- a/src/resolvers.js +++ b/src/resolvers.js @@ -1,24 +1,6 @@ import { GraphQLError } from 'graphql'; -function isIsoDate(value) { - const match = typeof value === 'string' && value.match(/^(\d{4}-\d{2}-\d{2})(?:T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2}))?$/); - if (!match) return false; - const calendarDate = new Date(`${match[1]}T00:00:00.000Z`); - return Number.isFinite(calendarDate.getTime()) && calendarDate.toISOString().startsWith(match[1]) && Number.isFinite(new Date(value).getTime()); -} - -function validateFinanceArgs(args, { requireTaxYear = false } = {}) { - const issues = []; - if (args.from !== undefined && args.from !== null && !isIsoDate(args.from)) issues.push('from must be a valid ISO-8601 date'); - if (args.to !== undefined && args.to !== null && !isIsoDate(args.to)) issues.push('to must be a valid ISO-8601 date'); - if (isIsoDate(args.from) && isIsoDate(args.to) && new Date(args.from).getTime() > new Date(args.to).getTime()) { - issues.push('from must be earlier than or equal to to'); - } - if (args.limit !== undefined && args.limit !== null && args.limit < 0) issues.push('limit must be greater than or equal to 0'); - if (args.offset !== undefined && args.offset !== null && args.offset < 0) issues.push('offset must be greater than or equal to 0'); - if (requireTaxYear && (args.taxYear < 1900 || args.taxYear > 9999)) issues.push('taxYear must be between 1900 and 9999'); - return issues; -} +import { validateFinanceArgs } from './validation/financeArgs.js'; async function runFinanceResolver(name, args, context, resolve, options = {}) { const issues = validateFinanceArgs(args, options); diff --git a/src/validation/financeArgs.js b/src/validation/financeArgs.js new file mode 100644 index 0000000..0c8c85e --- /dev/null +++ b/src/validation/financeArgs.js @@ -0,0 +1,32 @@ +const GRAPHQL_INT_MIN = -2147483648; +const GRAPHQL_INT_MAX = 2147483647; + +export function isGraphQLInt(value) { + return Number.isInteger(value) && value >= GRAPHQL_INT_MIN && value <= GRAPHQL_INT_MAX; +} + +export function isIsoDate(value) { + const match = typeof value === 'string' && value.match(/^(\d{4}-\d{2}-\d{2})(?:T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2}))?$/); + if (!match) return false; + const calendarDate = new Date(`${match[1]}T00:00:00.000Z`); + return Number.isFinite(calendarDate.getTime()) && calendarDate.toISOString().startsWith(match[1]) && Number.isFinite(new Date(value).getTime()); +} + +export function validateFinanceArgs(args, { requireTaxYear = false } = {}) { + const issues = []; + if (args.from !== undefined && args.from !== null && !isIsoDate(args.from)) issues.push('from must be a valid ISO-8601 date'); + if (args.to !== undefined && args.to !== null && !isIsoDate(args.to)) issues.push('to must be a valid ISO-8601 date'); + if (isIsoDate(args.from) && isIsoDate(args.to) && new Date(args.from).getTime() > new Date(args.to).getTime()) { + issues.push('from must be earlier than or equal to to'); + } + if (args.limit !== undefined && args.limit !== null && (!isGraphQLInt(args.limit) || args.limit < 0)) { + issues.push('limit must be a non-negative integer'); + } + if (args.offset !== undefined && args.offset !== null && (!isGraphQLInt(args.offset) || args.offset < 0)) { + issues.push('offset must be a non-negative integer'); + } + if (requireTaxYear && (!isGraphQLInt(args.taxYear) || args.taxYear < 1900 || args.taxYear > 9999)) { + issues.push('taxYear must be between 1900 and 9999'); + } + return issues; +} diff --git a/test/export.test.js b/test/export.test.js index 7eb0eb7..aecf0de 100644 --- a/test/export.test.js +++ b/test/export.test.js @@ -163,6 +163,29 @@ describe('export datasets', () => { await assert.rejects(() => loadDataset('tax-estimate', {}, { store, finance }), /taxYear/); }); + it('rejects invalid finance query parameters before loading data', async () => { + const calls = []; + finance = { + portfolioOverview: async () => { + calls.push('portfolio'); + }, + tradeHistory: async () => { + calls.push('trades'); + }, + taxEstimate: async () => { + calls.push('tax'); + }, + }; + + await assert.rejects(() => loadDataset('trades', { limit: 'abc' }, { store, finance }), { statusCode: 400 }); + await assert.rejects(() => loadDataset('trades', { limit: '2.5' }, { store, finance }), { statusCode: 400 }); + await assert.rejects(() => loadDataset('trades', { offset: '-1' }, { store, finance }), { statusCode: 400 }); + await assert.rejects(() => loadDataset('portfolio', { from: 'not-a-date' }, { store, finance }), { statusCode: 400 }); + await assert.rejects(() => loadDataset('tax-estimate', { taxYear: '2024x' }, { store, finance }), { statusCode: 400 }); + + assert.deepEqual(calls, []); + }); + it('rejects unknown datasets', async () => { await assert.rejects(() => loadDataset('nope', {}, { store, finance }), (error) => { assert.equal(error.statusCode, 404); @@ -194,6 +217,17 @@ describe('export router', () => { assert.match(readZip(response.body).get('xl/worksheets/sheet1.xml'), /Ada Lovelace/); }); + it('rejects invalid query parameters and oversized exports', async () => { + const invalidLimit = await callRouter(router, { path: '/trades', query: { limit: 'abc' } }); + const tinyRouter = createExportRouter({ store: createStore(), finance: createFinanceService(), maxExportRows: 1 }); + const tooLarge = await callRouter(tinyRouter, { path: '/users' }); + + assert.equal(invalidLimit.status, 400); + assert.match(invalidLimit.body.error, /limit/); + assert.equal(tooLarge.status, 413); + assert.match(tooLarge.body.error, /exceeds the limit/); + }); + it('returns 404 for unknown datasets and 400 for missing parameters', async () => { const unknown = await callRouter(router, { path: '/unknown' }); const missingYear = await callRouter(router, { path: '/tax-estimate' }); From 17545b2b84669dd55fcbfbf6a4f38a3edb639e82 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:27:13 +0000 Subject: [PATCH 5/9] Preserve query parsing for all exports Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- src/export/datasets.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/export/datasets.js b/src/export/datasets.js index 6a961d9..ada166d 100644 --- a/src/export/datasets.js +++ b/src/export/datasets.js @@ -8,7 +8,7 @@ import { isGraphQLInt, validateFinanceArgs } from '../validation/financeArgs.js'; /** Parses the shared pagination/filter query parameters of an export request. */ -function readParams(params = {}, options = {}) { +function readParams(params = {}, options = {}, { validate = true } = {}) { const issues = []; const optionalInt = (name) => { if (!Object.hasOwn(params, name) || params[name] === undefined || params[name] === null) return undefined; @@ -50,7 +50,7 @@ function readParams(params = {}, options = {}) { taxYear: optionalInt('taxYear'), }; - issues.push(...validateFinanceArgs(parsed, options)); + if (validate) issues.push(...validateFinanceArgs(parsed, options)); if (issues.length > 0) { throw Object.assign(new Error(`invalid export query parameters: ${issues.join('; ')}`), { statusCode: 400 }); } @@ -252,7 +252,9 @@ export async function loadDataset(name, params, context) { throw Object.assign(new Error(`unknown dataset "${name}"`), { statusCode: 404, datasets: datasetNames }); } - const parsedParams = dataset.financeArgs ? readParams(params, dataset.financeArgs === true ? {} : dataset.financeArgs) : {}; + const parsedParams = readParams(params, dataset.financeArgs === true ? {} : dataset.financeArgs || {}, { + validate: Boolean(dataset.financeArgs), + }); const result = await dataset.load(parsedParams, context); return { filename: dataset.filename, ...result }; } From 1aa296a481e026107f25f72ef224dcb1d881182f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:28:02 +0000 Subject: [PATCH 6/9] Simplify export param validation config Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- src/export/datasets.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/export/datasets.js b/src/export/datasets.js index ada166d..139436a 100644 --- a/src/export/datasets.js +++ b/src/export/datasets.js @@ -8,7 +8,7 @@ import { isGraphQLInt, validateFinanceArgs } from '../validation/financeArgs.js'; /** Parses the shared pagination/filter query parameters of an export request. */ -function readParams(params = {}, options = {}, { validate = true } = {}) { +function readParams(params = {}, { validate = false, requireTaxYear = false } = {}) { const issues = []; const optionalInt = (name) => { if (!Object.hasOwn(params, name) || params[name] === undefined || params[name] === null) return undefined; @@ -50,7 +50,7 @@ function readParams(params = {}, options = {}, { validate = true } = {}) { taxYear: optionalInt('taxYear'), }; - if (validate) issues.push(...validateFinanceArgs(parsed, options)); + if (validate) issues.push(...validateFinanceArgs(parsed, { requireTaxYear })); if (issues.length > 0) { throw Object.assign(new Error(`invalid export query parameters: ${issues.join('; ')}`), { statusCode: 400 }); } @@ -176,7 +176,7 @@ export const datasets = { portfolio: { filename: 'portfolio-overview', - financeArgs: true, + financeArgs: { validate: true }, async load(params, { finance }) { const overview = await finance.portfolioOverview(params); return { @@ -192,7 +192,7 @@ export const datasets = { trades: { filename: 'trade-history', - financeArgs: true, + financeArgs: { validate: true }, async load(params, { finance }) { const history = await finance.tradeHistory(params); return { @@ -208,7 +208,7 @@ export const datasets = { 'tax-estimate': { filename: 'tax-estimate', - financeArgs: { requireTaxYear: true }, + financeArgs: { validate: true, requireTaxYear: true }, async load(params, { finance }) { if (params.taxYear === undefined) { throw Object.assign(new Error('taxYear query parameter is required'), { statusCode: 400 }); @@ -252,9 +252,7 @@ export async function loadDataset(name, params, context) { throw Object.assign(new Error(`unknown dataset "${name}"`), { statusCode: 404, datasets: datasetNames }); } - const parsedParams = readParams(params, dataset.financeArgs === true ? {} : dataset.financeArgs || {}, { - validate: Boolean(dataset.financeArgs), - }); + const parsedParams = readParams(params, dataset.financeArgs); const result = await dataset.load(parsedParams, context); return { filename: dataset.filename, ...result }; } From 28db5e6b8897b64e4c19605482a688f1926674fb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:29:12 +0000 Subject: [PATCH 7/9] Refine export validation and row bounds Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- src/export/datasets.js | 37 +++++++++++++++++++++++++++---------- src/export/router.js | 10 +--------- test/export.test.js | 3 ++- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/export/datasets.js b/src/export/datasets.js index 139436a..bac6775 100644 --- a/src/export/datasets.js +++ b/src/export/datasets.js @@ -8,23 +8,26 @@ import { isGraphQLInt, validateFinanceArgs } from '../validation/financeArgs.js'; /** Parses the shared pagination/filter query parameters of an export request. */ -function readParams(params = {}, { validate = false, requireTaxYear = false } = {}) { +function readParams(params = {}, { datasetName = 'export', validate = false, requireTaxYear = false } = {}) { const issues = []; + const addIssue = (message) => { + if (validate) issues.push(message); + }; const optionalInt = (name) => { if (!Object.hasOwn(params, name) || params[name] === undefined || params[name] === null) return undefined; const raw = params[name]; if (Array.isArray(raw) || raw === '') { - issues.push(`${name} must be an integer`); + addIssue(`${name} must be an integer`); return undefined; } const value = String(raw); if (!/^-?\d+$/.test(value)) { - issues.push(`${name} must be an integer`); + addIssue(`${name} must be an integer`); return undefined; } const parsed = Number(value); if (!isGraphQLInt(parsed)) { - issues.push(`${name} must be a 32-bit integer`); + addIssue(`${name} must be a 32-bit integer`); return undefined; } return parsed; @@ -32,7 +35,7 @@ function readParams(params = {}, { validate = false, requireTaxYear = false } = const optionalString = (name) => { if (!Object.hasOwn(params, name) || params[name] === undefined || params[name] === null) return undefined; if (Array.isArray(params[name])) { - issues.push(`${name} must be a string`); + addIssue(`${name} must be a string`); return undefined; } return String(params[name]); @@ -52,12 +55,22 @@ function readParams(params = {}, { validate = false, requireTaxYear = false } = if (validate) issues.push(...validateFinanceArgs(parsed, { requireTaxYear })); if (issues.length > 0) { - throw Object.assign(new Error(`invalid export query parameters: ${issues.join('; ')}`), { statusCode: 400 }); + throw Object.assign(new Error(`invalid query parameters for ${datasetName} export: ${issues.join('; ')}`), { statusCode: 400 }); } return parsed; } +function countRows(sheets) { + return sheets.reduce((total, sheet) => total + (sheet.rows?.length ?? 0), 0); +} + +function enforceRowLimit(rowCount, maxRows) { + if (maxRows !== undefined && rowCount > maxRows) { + throw Object.assign(new Error(`export contains ${rowCount} rows, which exceeds the limit of ${maxRows}`), { statusCode: 413 }); + } +} + const USER_COLUMNS = [ { key: 'id', header: 'Id' }, { key: 'name', header: 'Name' }, @@ -142,8 +155,9 @@ const TAX_EVENT_COLUMNS = [ export const datasets = { users: { filename: 'users', - async load(_params, { store }) { + async load(_params, { store, maxExportRows }) { const users = store.listUsers(); + enforceRowLimit(users.length, maxExportRows); return { sheets: [ { @@ -158,13 +172,15 @@ export const datasets = { posts: { filename: 'posts', - async load(_params, { store }) { + async load(_params, { store, maxExportRows }) { + const posts = store.listPosts(); + enforceRowLimit(posts.length, maxExportRows); return { sheets: [ { name: 'Posts', columns: POST_COLUMNS, - rows: store.listPosts().map((post) => ({ + rows: posts.map((post) => ({ ...post, authorName: store.getUser(post.authorId)?.name ?? '', })), @@ -252,7 +268,8 @@ export async function loadDataset(name, params, context) { throw Object.assign(new Error(`unknown dataset "${name}"`), { statusCode: 404, datasets: datasetNames }); } - const parsedParams = readParams(params, dataset.financeArgs); + const parsedParams = readParams(params, { datasetName: name, ...dataset.financeArgs }); const result = await dataset.load(parsedParams, context); + enforceRowLimit(countRows(result.sheets), context.maxExportRows); return { filename: dataset.filename, ...result }; } diff --git a/src/export/router.js b/src/export/router.js index 8a4e78c..a15f6c4 100644 --- a/src/export/router.js +++ b/src/export/router.js @@ -13,10 +13,6 @@ const ERROR_COLUMNS = [ ]; const DEFAULT_MAX_EXPORT_ROWS = 10000; -function countRows(sheets) { - return sheets.reduce((total, sheet) => total + (sheet.rows?.length ?? 0), 0); -} - /** * Express router exposing spreadsheet downloads. * @@ -36,14 +32,10 @@ export function createExportRouter({ store, finance, logger, maxExportRows = DEF const name = req.params.dataset.replace(/\.xlsx$/i, ''); try { - const { filename, sheets, errors = [] } = await loadDataset(name, req.query, { store, finance }); + const { filename, sheets, errors = [] } = await loadDataset(name, req.query, { store, finance, maxExportRows }); // Partial finance results carry upstream errors; surface them in the // workbook instead of silently shipping incomplete data. const allSheets = errors.length > 0 ? [...sheets, { name: 'Errors', columns: ERROR_COLUMNS, rows: errors }] : sheets; - const rowCount = countRows(allSheets); - if (rowCount > maxExportRows) { - throw Object.assign(new Error(`export contains ${rowCount} rows, which exceeds the limit of ${maxExportRows}`), { statusCode: 413 }); - } const workbook = buildWorkbook({ sheets: allSheets }); res.set('content-type', XLSX_CONTENT_TYPE); diff --git a/test/export.test.js b/test/export.test.js index aecf0de..94687bd 100644 --- a/test/export.test.js +++ b/test/export.test.js @@ -135,7 +135,7 @@ describe('export datasets', () => { }); it('exports users with their post counts', async () => { - const { sheets } = await loadDataset('users', {}, { store, finance }); + const { sheets } = await loadDataset('users', { from: 'not-a-date', limit: 'abc' }, { store, finance }); assert.equal(sheets.length, 1); assert.equal(sheets[0].rows.length, 2); @@ -224,6 +224,7 @@ describe('export router', () => { assert.equal(invalidLimit.status, 400); assert.match(invalidLimit.body.error, /limit/); + assert.match(invalidLimit.body.error, /trades export/); assert.equal(tooLarge.status, 413); assert.match(tooLarge.body.error, /exceeds the limit/); }); From 89fcf49d2ca13007b62edb843cb2ead7bd0669de Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:29:58 +0000 Subject: [PATCH 8/9] Remove redundant export validation checks Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- src/export/datasets.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/export/datasets.js b/src/export/datasets.js index bac6775..b1eac97 100644 --- a/src/export/datasets.js +++ b/src/export/datasets.js @@ -155,9 +155,8 @@ const TAX_EVENT_COLUMNS = [ export const datasets = { users: { filename: 'users', - async load(_params, { store, maxExportRows }) { + async load(_params, { store }) { const users = store.listUsers(); - enforceRowLimit(users.length, maxExportRows); return { sheets: [ { @@ -172,9 +171,8 @@ export const datasets = { posts: { filename: 'posts', - async load(_params, { store, maxExportRows }) { + async load(_params, { store }) { const posts = store.listPosts(); - enforceRowLimit(posts.length, maxExportRows); return { sheets: [ { @@ -226,10 +224,6 @@ export const datasets = { filename: 'tax-estimate', financeArgs: { validate: true, requireTaxYear: true }, async load(params, { finance }) { - if (params.taxYear === undefined) { - throw Object.assign(new Error('taxYear query parameter is required'), { statusCode: 400 }); - } - const summary = await finance.taxEstimate(params); return { sheets: [ From 24a8debaced9a0b1929b90726597da0cb339763c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 16 Sep 2026 02:30:45 +0000 Subject: [PATCH 9/9] Restore missing taxYear validation message Co-authored-by: charles2ke <6725706+charles2ke@users.noreply.github.com> --- src/validation/financeArgs.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/validation/financeArgs.js b/src/validation/financeArgs.js index 0c8c85e..5178564 100644 --- a/src/validation/financeArgs.js +++ b/src/validation/financeArgs.js @@ -25,7 +25,9 @@ export function validateFinanceArgs(args, { requireTaxYear = false } = {}) { if (args.offset !== undefined && args.offset !== null && (!isGraphQLInt(args.offset) || args.offset < 0)) { issues.push('offset must be a non-negative integer'); } - if (requireTaxYear && (!isGraphQLInt(args.taxYear) || args.taxYear < 1900 || args.taxYear > 9999)) { + if (requireTaxYear && (args.taxYear === undefined || args.taxYear === null)) { + issues.push('taxYear query parameter is required'); + } else if (requireTaxYear && (!isGraphQLInt(args.taxYear) || args.taxYear < 1900 || args.taxYear > 9999)) { issues.push('taxYear must be between 1900 and 9999'); } return issues;