From 56ea43bdd89a7b5feabb3de7a1e2d13832c73763 Mon Sep 17 00:00:00 2001 From: Tom Unger Date: Thu, 27 Aug 2026 16:30:26 -0700 Subject: [PATCH 1/2] feat: import group expenses from a CSV file Adds a CSV import screen for a group, reachable from the group info drawer. Every selected row becomes an expense split equally between all group members, ready to be edited afterwards like any other expense. Columns are auto-detected from the header row and adjustable. A SplitPro expense has one description, but exports commonly spread it over several fields, so description accepts any number of columns and joins them in order, skipping those blank on a given row. The file's own categories are matched onto SplitPro's fixed list: the value is tried whole, then split on ':', '.' and '-', preferring a specific item over a broad section. Anything unrecognised falls back to the default category rather than failing the row. Exports disagree about which side of zero means spending, so the sign convention is selectable. Rows carrying the opposite sign are money received and import as negative expenses, moving balances the other way. Rows that look like a repeat of an existing group expense are flagged and start unselected, since a monthly workflow makes re-importing easy to do by accident. No server changes. The equal split reuses calculateParticipantSplit and rows are submitted in batches through the existing array-accepting addOrEditExpense mutation, the same path the bank-transaction import already uses. Co-Authored-By: Claude Opus 5 --- README.md | 4 +- docs/CSV_IMPORT.md | 142 +++++ public/locales/en/common.json | 39 ++ .../group/ImportExpensesFromCsv.tsx | 533 ++++++++++++++++++ src/components/group/importExpense.ts | 61 ++ src/lib/category.test.ts | 80 +++ src/lib/category.ts | 43 ++ src/lib/csv.test.ts | 51 ++ src/lib/csv.ts | 65 +++ src/lib/csvImport.test.ts | 379 +++++++++++++ src/lib/csvImport.ts | 303 ++++++++++ src/pages/groups/[groupId].tsx | 8 + src/pages/groups/[groupId]/import.tsx | 67 +++ src/tests/importExpense.test.ts | 213 +++++++ 14 files changed, 1987 insertions(+), 1 deletion(-) create mode 100644 docs/CSV_IMPORT.md create mode 100644 src/components/group/ImportExpensesFromCsv.tsx create mode 100644 src/components/group/importExpense.ts create mode 100644 src/lib/category.test.ts create mode 100644 src/lib/csv.test.ts create mode 100644 src/lib/csv.ts create mode 100644 src/lib/csvImport.test.ts create mode 100644 src/lib/csvImport.ts create mode 100644 src/pages/groups/[groupId]/import.tsx create mode 100644 src/tests/importExpense.test.ts diff --git a/README.md b/README.md index ca9887769..010decb32 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ See [docker/README.md](docker/README.md) and [docs/CONFIGURATION.md](docs/CONFIG - Negative expenses are supported for refunds and corrections. - PWA support with push notifications. - Activity feed with edits and deletions. +- Bulk import of a group's expenses from a CSV export, mapping columns and categories. - Detailed balances per person and per group. ## UI preview @@ -52,7 +53,7 @@ Groups are the primary way to use SplitPro. You can invite friends by email, or ### 4) Data utilities -Splitwise import supports friends and groups (partial import). Expenses themselves are not imported yet. You can export data from the balances view and account settings. +Splitwise import supports friends and groups (partial import); Splitwise expenses themselves are not imported yet. A group's expenses can be bulk imported from a CSV export, such as a monthly statement from a bank or a personal finance app. Columns are detected from the header row, the file's own categories are matched onto SplitPro's, and rows carrying the opposite sign are imported as money received. See [docs/CSV_IMPORT.md](docs/CSV_IMPORT.md). You can export data from the balances view and account settings. ### 5) Translations @@ -95,6 +96,7 @@ Bank integration allows you to load transactions from providers like Plaid and c - [docs/CURRENCY_CONVERSIONS.md](docs/CURRENCY_CONVERSIONS.md) - [docs/RECURRING_TRANSACTIONS.md](docs/RECURRING_TRANSACTIONS.md) - [docs/BANK_TRANSACTIONS.md](docs/BANK_TRANSACTIONS.md) +- [docs/CSV_IMPORT.md](docs/CSV_IMPORT.md) - [docker/README.md](docker/README.md) ## Versions diff --git a/docs/CSV_IMPORT.md b/docs/CSV_IMPORT.md new file mode 100644 index 000000000..42fdced13 --- /dev/null +++ b/docs/CSV_IMPORT.md @@ -0,0 +1,142 @@ +# CSV IMPORT + +SplitPro can import a group's expenses in bulk from a CSV file, for example a monthly export from +a bank or a personal finance app. No configuration or external service is required. + +## How to use it + +1. Open a group, then the group info drawer (the ⓘ icon). +2. Under **Actions**, choose **Import expenses from CSV**. +3. Pick the file. SplitPro reads the header row and guesses which columns hold the date, + the description and the amount. +4. Check the options and the preview, then import. + +## What gets created + +Every selected row becomes one expense in the group: + +- Split **equally** between all group members. +- Paid by whoever is selected in **Paid by** (the importing user by default). +- Categorised from the file's own category column where it can be matched, and **General** otherwise. +- Dated from the file's date column. + +Imported expenses are ordinary expenses. Nothing about them is special afterwards, and each one can +be edited or deleted individually. + +## Columns + +Only these columns are used; everything else in the file is ignored. Category is optional. + +| Role | Recognised header names | +| ------------------------- | ----------------------------------------------------------------------------------------------- | +| Date | `Transaction Date`, `Date`, `Posted`, `Day` | +| Description (one or more) | `Payee`, `Description`, `Merchant`, `Narrative`, `Details`, `Note`, `Memo`, `Reference`, `Name` | +| Amount | `Amount`, `Value`, `Debit`, `Total`, `Sum` | + +Detection is case-insensitive and matches substrings, so `Transaction Payee` is recognised as the +description. Whatever is detected can be changed, so a file with unrecognised headers still works. + +### Combining columns into the description + +A SplitPro expense has a single description, but exports often spread it over several fields -- +a payee plus a note, or a merchant plus a memo. **Description takes any number of columns**, chosen +with tick boxes rather than a dropdown. Every matching column is ticked automatically, and you can +add or remove any of them. + +Ticked columns are joined in CSV order, separated by `-`. Columns that are blank on a given row +are skipped, so a mostly-empty note column costs nothing: + +| Payee | Note | Imported description | +| -------------------- | ------------- | ---------------------------------- | +| `Whole Foods Market` | `weekly shop` | `Whole Foods Market - weekly shop` | +| `Haggen` | | `Haggen` | + +A row where every ticked column is blank is flagged as missing a description and cannot be +selected. + +## Categories + +SplitPro has a fixed category list, so the file's own categories are matched onto it. The value is +tried whole first, then split on `:`, `.` and `-` and each part tried in turn. A specific item beats +a broad section, and anything unrecognised falls back to **General** rather than failing the row. + +| In the file | Imported as | +| ------------------------ | ---------------------------------------------------- | +| `Food:Groceries` | Groceries | +| `Pets:Pet Supplies` | Pets | +| `Utilities` | Utilities | +| `Utilities:Web Services` | Utilities (the leaf is unknown, the section matches) | +| `Dining-Out` | Dining Out (matched before splitting) | +| `Widgets` | General | + +Matching ignores case, spaces and punctuation, so `Dining Out`, `dining-out` and `diningOut` all +resolve to the same category. The preview shows each row's resolved category as an icon, so a file +whose categories do not match is obvious before you import. + +Leave the category column unmapped to file everything under General. + +## Amount sign and money received + +Exports disagree about which side of zero means spending. The **Amount sign** option says which +convention the file uses: + +- **Expenses are negative** — `-$118.24` is money spent. This is the common bank convention. +- **Expenses are positive** — `118.24` is money spent. + +Rows with the **opposite** sign are money received — a refund, a rebate, a reimbursement — and are +imported as negative expenses, which move balances the other way. They are labelled `Received` in +the preview. + +Getting this option backwards is not silent: every row flips between expense and received in the +preview before anything is written. + +## Dates + +The date layout is detected from the file and can be overridden. `7/27/26` is unambiguously +month-first because 27 cannot be a month; a file where every row is ambiguous (`7/5/26`) defaults to +month-first, so check the preview if your export is day-first. + +Month names are matched in English only. For other languages, export dates in a numeric or ISO +format. + +## Duplicates + +Re-importing a file you have already imported is an easy mistake to make with a monthly workflow. +Rows matching an existing expense in the group on **date, amount, description and currency** are +labelled `Already in group` and start unselected. This is advisory — you can select them anyway. + +## Amount format + +Currency symbols and thousands separators are stripped, so `-$1,234.56` reads correctly. Numbers are +interpreted using the **separators of your app language**: with English selected, `1.234,56` is not +read as 1234.56. The preview shows the parsed value, so a mismatch is visible before importing. + +Accounting-style parentheses for negatives (`(118.24)`) are not recognised as a sign; such a value +reads as positive. Export plain signed numbers instead. + +One currency applies to the whole import; there is no per-row currency column. + +## File format + +Standard RFC 4180 CSV: + +- Comma separated, first line is the header. +- Fields may be quoted; quoted fields may contain commas, newlines and `""`-escaped quotes. +- LF and CRLF line endings are both accepted, as is a leading byte order mark. +- Blank lines are ignored. + +Rows that cannot be used are listed in the preview with a reason (unreadable date, unreadable +amount, zero amount, missing description) and cannot be selected. + +## Example + +```csv +Date,Chk #,Transaction Payee,Note,Account,Category,Amount +7/27/26,0,Whole Foods Market,,Apple Card,Food:Groceries,-$118.24 +7/28/26,0,Haggen,,Apple Card,Food:Groceries,-$30.04 +7/13/26,0,Payment to xfinity,,Checking (0967),Utilities,-$55.00 +8/2/26,0,Utility rebate,,Checking (0967),Utilities,$40.00 +``` + +With **Expenses are negative**, the first three rows import as expenses and the rebate imports as +money received. diff --git a/public/locales/en/common.json b/public/locales/en/common.json index 5ec3f196f..4b747f1c0 100644 --- a/public/locales/en/common.json +++ b/public/locales/en/common.json @@ -312,6 +312,45 @@ "title": "Group statistics", "total_expenses": "Total expenses" }, + "import_csv": { + "title": "Import expenses from CSV", + "note": "Every row becomes an expense split equally between all group members, filed under General so you can categorise it later.", + "choose_file": "Choose file", + "no_file_chosen": "No file chosen", + "columns": "Columns", + "column_date": "Date", + "column_description": "Description", + "column_description_hint": "Tick one or more columns. They are joined in order, and blank ones are skipped.", + "column_amount": "Amount", + "column_category": "Category", + "column_unmapped": "Not mapped", + "options": "Options", + "amount_sign": "Amount sign", + "amount_sign_options": { + "expenses_negative": "Expenses are negative", + "expenses_positive": "Expenses are positive" + }, + "date_format": "Date format", + "paid_by": "Paid by", + "currency": "Currency", + "preview": "Preview", + "selected": "{{selected}} of {{total}} selected", + "received": "Received", + "already_in_group": "Already in group", + "no_name": "No description", + "importing": "Imported {{done}} of {{total}}", + "row_errors": { + "invalid_date": "Unreadable date", + "invalid_amount": "Unreadable amount", + "zero_amount": "Zero amount", + "missing_name": "Missing description" + }, + "messages": { + "import_success": "Imported {{count}} expenses", + "import_partial": "Imported {{count}} expenses before the import failed", + "no_rows_found": "That file has no rows below the header" + } + }, "messages": { "default_split_cleared": "Default split cleared", "default_split_updated": "Default split updated", diff --git a/src/components/group/ImportExpensesFromCsv.tsx b/src/components/group/ImportExpensesFromCsv.tsx new file mode 100644 index 000000000..a068f8ec9 --- /dev/null +++ b/src/components/group/ImportExpensesFromCsv.tsx @@ -0,0 +1,533 @@ +import { PaperClipIcon } from '@heroicons/react/24/solid'; +import { type User } from '@prisma/client'; +import { useRouter } from 'next/router'; +import React, { useCallback, useMemo, useState } from 'react'; +import { toast } from 'sonner'; + +import { useTranslationWithUtils } from '~/hooks/useTranslationWithUtils'; +import { parseCsv } from '~/lib/csv'; +import { + AMOUNT_SIGNS, + type AmountSign, + type ColumnMapping, + type CsvColumn, + DATE_FORMATS, + type DateFormat, + type ParsedRow, + detectColumns, + detectDateFormat, + findDuplicateLines, + isAmountSign, + isDateFormat, + parseRows, +} from '~/lib/csvImport'; +import { type CurrencyCode, parseCurrencyCode } from '~/lib/currency'; +import { cn } from '~/lib/utils'; +import { api } from '~/utils/api'; + +import { CurrencyPicker } from '../AddExpense/CurrencyPicker'; +import { CategoryIcon } from '../ui/categoryIcons'; +import { Button } from '../ui/button'; +import { Checkbox } from '../ui/checkbox'; +import { Input } from '../ui/input'; +import { NativeSelect, NativeSelectOption } from '../ui/native-select'; +import { Separator } from '../ui/separator'; +import { LoadingSpinner } from '../ui/spinner'; +import { buildImportedExpense } from './importExpense'; + +/** Roles filled by exactly one column. Description is chosen separately, and takes several. */ +const SINGLE_COLUMNS = ['date', 'amount', 'category'] as const satisfies readonly CsvColumn[]; + +/** Sentinel for the "not mapped" option, since a select cannot hold null. */ +const UNMAPPED = ''; + +/** Expenses are sent in batches so that a long file is not one oversized request. */ +const IMPORT_BATCH_SIZE = 25; + +const chunk = (items: T[], size: number): T[][] => + items.reduce((acc, item, index) => { + if (0 === index % size) { + acc.push([]); + } + acc[acc.length - 1]!.push(item); + return acc; + }, []); + +export const ImportExpensesFromCsv: React.FC<{ + groupId: number; + user: { id: number; currency?: string | null; defaultCurrency?: string | null }; +}> = ({ groupId, user }) => { + const { t, i18n, displayName, getCurrencyHelpersCached } = useTranslationWithUtils(); + const router = useRouter(); + + const groupDetailQuery = api.group.getGroupDetails.useQuery({ groupId }); + const expensesQuery = api.expense.getGroupExpenses.useQuery({ groupId }); + const addExpenseMutation = api.expense.addOrEditExpense.useMutation(); + + const [fileName, setFileName] = useState(null); + const [headers, setHeaders] = useState([]); + const [records, setRecords] = useState([]); + const [mapping, setMapping] = useState({ + date: null, + amount: null, + category: null, + description: [], + }); + const [dateFormat, setDateFormat] = useState(DATE_FORMATS[0]); + const [amountSign, setAmountSign] = useState('expenses_negative'); + const [currency, setCurrency] = useState( + parseCurrencyCode(user.currency ?? user.defaultCurrency ?? 'USD'), + ); + const [paidById, setPaidById] = useState(user.id); + // Only rows the user has explicitly toggled; everything else follows the default. + const [overrides, setOverrides] = useState>({}); + const [importedCount, setImportedCount] = useState(0); + /* Tracked separately from the mutation: this stays true across the whole batch run, + where `isPending` would flicker between batches. */ + const [isImporting, setIsImporting] = useState(false); + + // Precomputed so the option keys do not have to be derived from the array index. + const headerOptions = useMemo( + () => headers.map((header, index) => ({ id: `${index}:${header}`, header, index })), + [headers], + ); + + const members = useMemo( + () => groupDetailQuery.data?.groupUsers.map((groupUser) => groupUser.user) ?? [], + [groupDetailQuery.data], + ); + + const rows = useMemo( + () => + parseRows({ + rows: records, + mapping, + dateFormat, + amountSign, + currency, + locale: i18n.language, + }), + [records, mapping, dateFormat, amountSign, currency, i18n.language], + ); + + const duplicateLines = useMemo( + () => + findDuplicateLines( + rows, + (expensesQuery.data ?? []).filter((expense) => expense.currency === currency), + ), + [rows, expensesQuery.data, currency], + ); + + const isIncluded = useCallback( + (row: ParsedRow) => + !row.error && (overrides[row.lineNumber] ?? !duplicateLines.has(row.lineNumber)), + [overrides, duplicateLines], + ); + + const selectedRows = useMemo(() => rows.filter(isIncluded), [rows, isIncluded]); + + const { toUIString } = getCurrencyHelpersCached(currency); + + const onFileChange = useCallback( + async (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + if (!file) { + return; + } + + try { + const parsed = parseCsv(await file.text()); + + if (0 === parsed.rows.length) { + toast.error(t('group_details.import_csv.messages.no_rows_found')); + return; + } + + const detected = detectColumns(parsed.headers); + const dateColumn = detected.date; + + setFileName(file.name); + setHeaders(parsed.headers); + setRecords(parsed.rows); + setMapping(detected); + setOverrides({}); + setImportedCount(0); + setDateFormat( + detectDateFormat( + null === dateColumn ? [] : parsed.rows.map((row) => row[dateColumn] ?? ''), + ), + ); + } catch (error) { + console.error(error); + toast.error(t('errors.import_failed')); + } + }, + [t], + ); + + const onColumnPick = useCallback( + (column: CsvColumn, value: string) => + setMapping((current) => ({ + ...current, + [column]: UNMAPPED === value ? null : Number(value), + })), + [], + ); + + const onDescriptionToggle = useCallback( + (index: number, checked: boolean) => + setMapping((current) => ({ + ...current, + // Kept in CSV order so the joined description reads the way the file is laid out. + description: checked + ? [...current.description, index].sort((a, b) => a - b) + : current.description.filter((i) => i !== index), + })), + [], + ); + + const onCurrencyPick = useCallback((picked: CurrencyCode | null) => { + if (picked) { + setCurrency(picked); + } + }, []); + + const onAmountSignChange = useCallback((event: React.ChangeEvent) => { + if (isAmountSign(event.target.value)) { + setAmountSign(event.target.value); + } + }, []); + + const onDateFormatChange = useCallback((event: React.ChangeEvent) => { + if (isDateFormat(event.target.value)) { + setDateFormat(event.target.value); + } + }, []); + + const onPaidByChange = useCallback( + (event: React.ChangeEvent) => setPaidById(Number(event.target.value)), + [], + ); + + const onRowToggle = useCallback( + (lineNumber: number, included: boolean) => + setOverrides((current) => ({ ...current, [lineNumber]: included })), + [], + ); + + const toExpense = useCallback( + (row: ParsedRow, expenseDate: Date, paidBy: User) => + buildImportedExpense({ row, expenseDate, paidBy, members, currency, groupId }), + [members, currency, groupId], + ); + + const onImport = useCallback(async () => { + const paidBy = members.find((member) => member.id === paidById); + + if (!paidBy) { + return; + } + + const expenses = selectedRows.flatMap((row) => + row.date ? [toExpense(row, row.date, paidBy)] : [], + ); + + if (0 === expenses.length) { + return; + } + + let imported = 0; + setIsImporting(true); + + try { + for (const batch of chunk(expenses, IMPORT_BATCH_SIZE)) { + await addExpenseMutation.mutateAsync(batch); + imported += batch.length; + setImportedCount(imported); + } + + toast.success(t('group_details.import_csv.messages.import_success', { count: imported })); + router.push(`/groups/${groupId}`).catch(console.error); + } catch (error) { + console.error(error); + setImportedCount(0); + toast.error( + 0 === imported + ? t('errors.import_failed') + : t('group_details.import_csv.messages.import_partial', { count: imported }), + ); + } finally { + setIsImporting(false); + } + }, [members, paidById, selectedRows, toExpense, addExpenseMutation, router, groupId, t]); + + const filePicker = ; + + if (!fileName) { + return ( + <> + {filePicker} +

{t('group_details.import_csv.note')}

+ + ); + } + + return ( + <> + {filePicker} + +

{t('group_details.import_csv.columns')}

+
+ {SINGLE_COLUMNS.map((column) => ( + + ))} + +
+

{t('group_details.import_csv.column_description')}

+

+ {t('group_details.import_csv.column_description_hint')} +

+
+ {headerOptions.map((option) => ( + + ))} +
+
+
+ +

{t('group_details.import_csv.options')}

+
+ + + + + + +
+ {t('group_details.import_csv.currency')} +
+ +
+
+
+ +
+

{t('group_details.import_csv.preview')}

+

+ {t('group_details.import_csv.selected', { + selected: selectedRows.length, + total: rows.length, + })} +

+
+ +
+ {rows.map((row) => ( + + ))} +
+ +
+ +
+ + ); +}; + +interface HeaderOption { + id: string; + header: string; + index: number; +} + +const DescriptionColumn: React.FC<{ + option: HeaderOption; + checked: boolean; + onToggle: (index: number, checked: boolean) => void; +}> = ({ option, checked, onToggle }) => { + const onCheckedChange = useCallback( + (state: boolean | 'indeterminate') => onToggle(option.index, true === state), + [option.index, onToggle], + ); + + return ( + + ); +}; + +const ColumnSelect: React.FC<{ + column: CsvColumn; + options: HeaderOption[]; + value: number | null; + onPick: (column: CsvColumn, value: string) => void; +}> = ({ column, options, value, onPick }) => { + const { t } = useTranslationWithUtils(); + + const onChange = useCallback( + (event: React.ChangeEvent) => onPick(column, event.target.value), + [column, onPick], + ); + + return ( + + ); +}; + +const PreviewRow: React.FC<{ + row: ParsedRow; + included: boolean; + isDuplicate: boolean; + amount: string; + onToggle: (lineNumber: number, included: boolean) => void; +}> = ({ row, included, isDuplicate, amount, onToggle }) => { + const { t, toUIDate } = useTranslationWithUtils(); + + const onCheckedChange = useCallback( + (checked: boolean | 'indeterminate') => onToggle(row.lineNumber, true === checked), + [row.lineNumber, onToggle], + ); + + const note = [ + row.date ? toUIDate(row.date, { year: true }) : row.raw.date, + row.error && t(`group_details.import_csv.row_errors.${row.error}`), + !row.error && 0n > row.amount && t('group_details.import_csv.received'), + !row.error && isDuplicate && t('group_details.import_csv.already_in_group'), + ] + .filter(Boolean) + .join(' · '); + + return ( + <> +
+
+ + +
+

{row.name || t('group_details.import_csv.no_name')}

+

{note}

+
+
+

row.amount ? 'text-green-500' : '', + )} + > + {amount} +

+
+ + + ); +}; + +const FilePicker: React.FC<{ + fileName: string | null; + onFileChange: (event: React.ChangeEvent) => void; +}> = ({ fileName, onFileChange }) => { + const { t } = useTranslationWithUtils(); + + return ( + + ); +}; diff --git a/src/components/group/importExpense.ts b/src/components/group/importExpense.ts new file mode 100644 index 000000000..a0c946fb9 --- /dev/null +++ b/src/components/group/importExpense.ts @@ -0,0 +1,61 @@ +import { SplitType, type User } from '@prisma/client'; + +import { type ParsedRow } from '~/lib/csvImport'; +import { type SplitShares, calculateParticipantSplit, initSplitShares } from '~/store/addStore'; +import { type CreateExpense } from '~/types/expense.types'; +import { BigMath } from '~/utils/numbers'; + +export interface BuildImportedExpenseOptions { + row: ParsedRow; + expenseDate: Date; + paidBy: User; + members: User[]; + currency: string; + groupId: number; +} + +/** + * Turns one parsed row into an expense split equally across the whole group. + * + * Money received is a negative expense: the magnitude is split as usual and then every + * amount is flipped, which is exactly what AddOrEditExpensePage does for a negative + * amount. The category comes from the row, already resolved to a SplitPro category. + */ +export const buildImportedExpense = ({ + row, + expenseDate, + paidBy, + members, + currency, + groupId, +}: BuildImportedExpenseOptions): CreateExpense => { + const isNegative = 0n > row.amount; + const sign = isNegative ? -1n : 1n; + + const splitShares = members.reduce((acc, member) => { + acc[member.id] = initSplitShares(); + return acc; + }, {}); + + const { participants } = calculateParticipantSplit({ + amount: BigMath.abs(row.amount), + participants: members, + splitType: SplitType.EQUAL, + splitShares, + paidBy, + expenseDate, + isNegative, + }); + + return { + name: row.name, + currency, + amount: row.amount, + groupId, + splitType: SplitType.EQUAL, + paidBy: paidBy.id, + participants: participants.map((p) => ({ userId: p.id, amount: (p.amount ?? 0n) * sign })), + category: row.category, + expenseDate, + }; +}; diff --git a/src/lib/category.test.ts b/src/lib/category.test.ts new file mode 100644 index 000000000..f1ed1ed23 --- /dev/null +++ b/src/lib/category.test.ts @@ -0,0 +1,80 @@ +import { DEFAULT_CATEGORY, matchCategory } from '~/lib/category'; + +describe('matchCategory', () => { + describe('DirectMatches', () => { + it('should match a section by name', () => { + expect(matchCategory('Utilities')).toBe('utilities'); + }); + + it('should match an item by name', () => { + expect(matchCategory('Groceries')).toBe('groceries'); + }); + + it('should ignore case and surrounding whitespace', () => { + expect(matchCategory(' gRoCeRiEs ')).toBe('groceries'); + }); + }); + + describe('NestedCategories', () => { + it('should split on a colon and prefer the specific item', () => { + expect(matchCategory('Food:Groceries')).toBe('groceries'); + }); + + it('should split on a dot', () => { + expect(matchCategory('Food.Groceries')).toBe('groceries'); + }); + + it('should split on a hyphen', () => { + expect(matchCategory('Food-Groceries')).toBe('groceries'); + }); + + it('should fall back to the section when the leaf is unknown', () => { + expect(matchCategory('Utilities:Telephone/Cellular')).toBe('utilities'); + expect(matchCategory('Utilities:Web Services')).toBe('utilities'); + }); + + it('should match an item that appears in a different section', () => { + // `pets` lives under `home`, but the export nests it under its own heading. + expect(matchCategory('Pets:Pet Supplies')).toBe('pets'); + }); + + it('should handle more than two levels', () => { + expect(matchCategory('Expenses:Travel:Hotel')).toBe('hotel'); + }); + }); + + describe('Normalisation', () => { + it('should match a multi-word item written with a space', () => { + expect(matchCategory('Dining Out')).toBe('diningOut'); + }); + + it('should match a multi-word item written with a hyphen', () => { + // The whole string is tried before splitting, so this still resolves. + expect(matchCategory('Dining-Out')).toBe('diningOut'); + }); + + it('should match a multi-word item written in camel case', () => { + expect(matchCategory('diningOut')).toBe('diningOut'); + }); + }); + + describe('Fallback', () => { + it('should fall back for an unknown category', () => { + expect(matchCategory('Widgets')).toBe(DEFAULT_CATEGORY); + }); + + it('should fall back for an empty value', () => { + expect(matchCategory('')).toBe(DEFAULT_CATEGORY); + expect(matchCategory(' ')).toBe(DEFAULT_CATEGORY); + }); + + it('should fall back for separators alone', () => { + expect(matchCategory(':::')).toBe(DEFAULT_CATEGORY); + }); + + it('should never resolve to the shared `other` placeholder', () => { + // `other` is not a storable value -- picking it stores the section name instead. + expect(matchCategory('Other')).toBe(DEFAULT_CATEGORY); + }); + }); +}); diff --git a/src/lib/category.ts b/src/lib/category.ts index 3596b2fbe..8d75fd5cd 100644 --- a/src/lib/category.ts +++ b/src/lib/category.ts @@ -26,3 +26,46 @@ type CategoryValues = (typeof CATEGORIES)[CategorySection][number]; type CategoryWithoutOther = Exclude; export type CategoryItem = CategoryWithoutOther | CategorySection; + +/** Separators used by finance exports to nest categories, e.g. `Food:Groceries`. */ +const CATEGORY_SEPARATORS = /[:.-]/; + +const normalize = (value: string) => value.toLowerCase().replace(/[^a-z0-9]/g, ''); + +const toLookup = (values: readonly string[]) => + new Map(values.map((value) => [normalize(value), value])); + +const SECTION_LOOKUP = toLookup(Object.keys(CATEGORIES)); + +/* `other` is excluded: picking it stores the section name, so it is not a value of its own. */ +const ITEM_LOOKUP = toLookup( + Object.values(CATEGORIES) + .flat() + .filter((item) => 'other' !== item), +); + +/** + * Maps a category from an external system onto a SplitPro category. + * + * The whole string is tried first, then each part after splitting on `:`, `.` and `-`, so + * both `Dining-Out` and `Food:Groceries` resolve. A specific item wins over a section -- + * `Food:Groceries` becomes `groceries`, not `food`. Anything unrecognised falls back to + * the default category rather than failing the row. + */ +export const matchCategory = (value: string): string => { + const candidates = [value, ...value.split(CATEGORY_SEPARATORS)] + .map(normalize) + .filter((candidate) => '' !== candidate); + + const item = candidates.find((candidate) => ITEM_LOOKUP.has(candidate)); + if (item) { + return ITEM_LOOKUP.get(item)!; + } + + const section = candidates.find((candidate) => SECTION_LOOKUP.has(candidate)); + if (section) { + return SECTION_LOOKUP.get(section)!; + } + + return DEFAULT_CATEGORY; +}; diff --git a/src/lib/csv.test.ts b/src/lib/csv.test.ts new file mode 100644 index 000000000..ca52820b2 --- /dev/null +++ b/src/lib/csv.test.ts @@ -0,0 +1,51 @@ +import { parseCsv } from '~/lib/csv'; + +describe('parseCsv', () => { + it('should split a plain record into fields', () => { + const { headers, rows } = parseCsv('a,b,c\n1,2,3\n'); + + expect(headers).toEqual(['a', 'b', 'c']); + expect(rows).toEqual([['1', '2', '3']]); + }); + + it('should keep separators inside quoted fields', () => { + const { rows } = parseCsv('a,b\n"Smith, John",5\n'); + + expect(rows).toEqual([['Smith, John', '5']]); + }); + + it('should unescape doubled quotes', () => { + const { rows } = parseCsv('a\n"He said ""hi"""\n'); + + expect(rows).toEqual([['He said "hi"']]); + }); + + it('should keep newlines inside quoted fields', () => { + const { rows } = parseCsv('a,b\n"line one\nline two",5\n'); + + expect(rows).toEqual([['line one\nline two', '5']]); + }); + + it('should handle CRLF endings and a byte order mark', () => { + const { headers, rows } = parseCsv('\uFEFFa,b\r\n1,2\r\n'); + + expect(headers).toEqual(['a', 'b']); + expect(rows).toEqual([['1', '2']]); + }); + + it('should drop blank lines rather than emit empty records', () => { + const { rows } = parseCsv('a,b\n1,2\n\n\n'); + + expect(rows).toEqual([['1', '2']]); + }); + + it('should preserve empty fields between separators', () => { + const { rows } = parseCsv('a,b,c\n1,,3\n'); + + expect(rows).toEqual([['1', '', '3']]); + }); + + it('should return nothing for empty input', () => { + expect(parseCsv('')).toEqual({ headers: [], rows: [] }); + }); +}); diff --git a/src/lib/csv.ts b/src/lib/csv.ts new file mode 100644 index 000000000..ac534ea1b --- /dev/null +++ b/src/lib/csv.ts @@ -0,0 +1,65 @@ +/** + * Minimal RFC 4180 CSV reader. + * + * Supports quoted fields containing separators, newlines and escaped (`""`) quotes, + * both LF and CRLF line endings, and a leading byte order mark. Fully blank lines + * are dropped, so trailing newlines do not produce empty records. + */ +export const parseCsv = (text: string): { headers: string[]; rows: string[][] } => { + const input = text.startsWith('\uFEFF') ? text.slice(1) : text; + + const records: string[][] = []; + let record: string[] = []; + let field = ''; + let quoted = false; + let index = 0; + + const endField = () => { + record.push(field); + field = ''; + }; + + const endRecord = () => { + endField(); + records.push(record); + record = []; + }; + + while (index < input.length) { + const char = input[index]!; + + if (quoted) { + if ('"' !== char) { + field += char; + index += 1; + } else if ('"' === input[index + 1]) { + // An escaped quote inside a quoted field. + field += '"'; + index += 2; + } else { + quoted = false; + index += 1; + } + } else if ('"' === char && '' === field) { + quoted = true; + index += 1; + } else if (',' === char) { + endField(); + index += 1; + } else if ('\n' === char || '\r' === char) { + endRecord(); + index += '\r' === char && '\n' === input[index + 1] ? 2 : 1; + } else { + field += char; + index += 1; + } + } + + if ('' !== field || 0 < record.length) { + endRecord(); + } + + const [headers = [], ...rows] = records.filter((r) => r.some((f) => '' !== f.trim())); + + return { headers: headers.map((h) => h.trim()), rows }; +}; diff --git a/src/lib/csvImport.test.ts b/src/lib/csvImport.test.ts new file mode 100644 index 000000000..cd4c209e2 --- /dev/null +++ b/src/lib/csvImport.test.ts @@ -0,0 +1,379 @@ +import { parseCsv } from '~/lib/csv'; +import { + type AmountSign, + type ColumnMapping, + DESCRIPTION_SEPARATOR, + detectColumns, + detectDateFormat, + findDuplicateLines, + parseDate, + parseRows, +} from '~/lib/csvImport'; + +const HEADERS = ['Date', 'Chk #', 'Transaction Payee', 'Note', 'Account', 'Category', 'Amount']; + +const MAPPING: ColumnMapping = { date: 0, description: [2], amount: 6, category: 5 }; + +const parseAmounts = (rows: string[][], amountSign: AmountSign) => + parseRows({ + rows, + mapping: MAPPING, + dateFormat: 'M/d/yy', + amountSign, + currency: 'USD', + locale: 'en-US', + }); + +describe('detectColumns', () => { + it('should map the columns of a typical finance export', () => { + // Both `Transaction Payee` and `Note` are description-ish, so both are picked up. + expect(detectColumns(HEADERS)).toEqual({ + date: 0, + description: [2, 3], + amount: 6, + category: 5, + }); + }); + + it('should prefer a payee column over a note column', () => { + expect(detectColumns(['Date', 'Note', 'Payee', 'Amount']).description).toEqual([1, 2]); + }); + + it('should recognise alternative header names', () => { + expect(detectColumns(['Posted', 'Description', 'Value'])).toEqual({ + date: 0, + description: [1], + amount: 2, + category: null, + }); + }); + + it('should never assign one column to two roles', () => { + const mapping = detectColumns(['Amount', 'Amount', 'Amount']); + + expect(mapping.amount).toBe(0); + expect(mapping.date).toBeNull(); + }); + + it('should return nulls when nothing matches', () => { + expect(detectColumns(['foo', 'bar'])).toEqual({ + date: null, + description: [], + amount: null, + category: null, + }); + }); +}); + +describe('description columns', () => { + const parseWith = (description: number[], row: string[]) => + parseRows({ + rows: [row], + mapping: { date: 0, description, amount: 6, category: null }, + dateFormat: 'M/d/yy', + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + })[0]; + + const row = ['7/27/26', '0', 'Whole Foods Market', 'weekly shop', 'Apple Card', 'Food', '-$1.00']; + + it('should use a single column on its own', () => { + expect(parseWith([2], row)?.name).toBe('Whole Foods Market'); + }); + + it('should join several columns in the order given', () => { + expect(parseWith([2, 3], row)?.name).toBe( + `Whole Foods Market${DESCRIPTION_SEPARATOR}weekly shop`, + ); + }); + + it('should respect the caller order rather than sorting', () => { + expect(parseWith([3, 2], row)?.name).toBe( + `weekly shop${DESCRIPTION_SEPARATOR}Whole Foods Market`, + ); + }); + + it('should skip columns that are blank on this row', () => { + const sparse = ['7/27/26', '0', 'Haggen', '', 'Apple Card', 'Food', '-$1.00']; + + expect(parseWith([2, 3], sparse)?.name).toBe('Haggen'); + }); + + it('should join three columns', () => { + expect(parseWith([2, 3, 4], row)?.name).toBe( + ['Whole Foods Market', 'weekly shop', 'Apple Card'].join(DESCRIPTION_SEPARATOR), + ); + }); + + it('should flag a row where every mapped column is blank', () => { + const blank = ['7/27/26', '0', '', '', 'Apple Card', 'Food', '-$1.00']; + + expect(parseWith([2, 3], blank)?.error).toBe('missing_name'); + }); + + it('should flag a row when no column is mapped at all', () => { + expect(parseWith([], row)?.error).toBe('missing_name'); + }); +}); + +describe('parseDate', () => { + it('should read a US short date', () => { + expect(parseDate('7/27/26', 'M/d/yy')).toEqual(new Date(2026, 6, 27)); + }); + + it('should reject a value the format does not describe exactly', () => { + // `date-fns` would otherwise read `26` as the year 26 AD. + expect(parseDate('7/27/26', 'M/d/yyyy')).toBeNull(); + expect(parseDate('07/27/26', 'M/d/yy')).toBeNull(); + }); + + it('should reject an impossible day-month pairing', () => { + expect(parseDate('7/27/26', 'd/M/yy')).toBeNull(); + }); + + it('should reject junk and blanks', () => { + expect(parseDate('not a date', 'M/d/yy')).toBeNull(); + expect(parseDate(' ', 'M/d/yy')).toBeNull(); + }); +}); + +describe('detectDateFormat', () => { + it('should pick month-first when a day exceeds twelve', () => { + expect(detectDateFormat(['7/27/26', '7/4/26'])).toBe('M/d/yy'); + }); + + it('should pick day-first when the first component exceeds twelve', () => { + expect(detectDateFormat(['27/7/26', '4/7/26'])).toBe('d/M/yy'); + }); + + it('should recognise ISO dates', () => { + expect(detectDateFormat(['2026-07-27'])).toBe('yyyy-MM-dd'); + }); + + it('should recognise zero-padded dates', () => { + expect(detectDateFormat(['07/27/2026'])).toBe('MM/dd/yyyy'); + }); + + it('should prefer the layout that fits the most samples', () => { + // The second value rules out month-first, so day-first reads both. + expect(detectDateFormat(['7/4/26', '27/7/26'])).toBe('d/M/yy'); + }); + + it('should tolerate a single unreadable sample', () => { + expect(detectDateFormat(['7/27/26', 'n/a', '7/28/26'])).toBe('M/d/yy'); + }); + + it('should fall back to the most common format when nothing fits', () => { + expect(detectDateFormat(['whenever'])).toBe('M/d/yy'); + }); +}); + +describe('parseRows', () => { + const row = (date: string, payee: string, amount: string) => [ + date, + '0', + payee, + '', + '', + '', + amount, + ]; + + it('should read a negative-is-spending file as expenses', () => { + const [parsed] = parseAmounts( + [row('7/27/26', 'Whole Foods Market', '-$118.24')], + 'expenses_negative', + ); + + expect(parsed).toMatchObject({ + lineNumber: 2, + name: 'Whole Foods Market', + amount: 11824n, + error: undefined, + }); + expect(parsed?.date).toEqual(new Date(2026, 6, 27)); + }); + + it('should read the opposite sign in the same file as money received', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Refund', '$40.00')], 'expenses_negative'); + + expect(parsed?.amount).toBe(-4000n); + }); + + it('should read a positive-is-spending file as expenses', () => { + const [parsed] = parseAmounts( + [row('7/27/26', 'Whole Foods Market', '118.24')], + 'expenses_positive', + ); + + expect(parsed?.amount).toBe(11824n); + }); + + it('should read the opposite sign in a positive-is-spending file as money received', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Refund', '-40.00')], 'expenses_positive'); + + expect(parsed?.amount).toBe(-4000n); + }); + + it('should strip currency symbols and thousands separators', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Rent', '-$1,234.56')], 'expenses_negative'); + + expect(parsed?.amount).toBe(123456n); + }); + + it('should number lines from the source file, header included', () => { + const parsed = parseAmounts( + [row('7/1/26', 'One', '-1.00'), row('7/2/26', 'Two', '-2.00')], + 'expenses_negative', + ); + + expect(parsed.map((p) => p.lineNumber)).toEqual([2, 3]); + }); + + it('should flag an unreadable date', () => { + const [parsed] = parseAmounts([row('nope', 'Shop', '-1.00')], 'expenses_negative'); + + expect(parsed?.error).toBe('invalid_date'); + }); + + it('should flag a missing amount', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Shop', '')], 'expenses_negative'); + + expect(parsed?.error).toBe('invalid_amount'); + }); + + it('should flag a zero amount', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Shop', '$0.00')], 'expenses_negative'); + + expect(parsed?.error).toBe('zero_amount'); + }); + + it('should flag a missing description', () => { + const [parsed] = parseAmounts([row('7/27/26', ' ', '-1.00')], 'expenses_negative'); + + expect(parsed?.error).toBe('missing_name'); + }); + + it('should tolerate an unmapped column', () => { + const [parsed] = parseRows({ + rows: [['7/27/26', '0', 'Shop', '', '', '', '-1.00']], + mapping: { date: 0, description: [], amount: 6, category: null }, + dateFormat: 'M/d/yy', + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + }); + + expect(parsed?.error).toBe('missing_name'); + }); + + it('should parse the sample export end to end', () => { + const csv = [ + 'Date,Chk #,Transaction Payee,Note,Account,Category,Amount', + '7/27/26,0,Whole Foods Market,,Apple Card,Food:Groceries,-$118.24', + '7/28/26,0,Haggen,,Apple Card,Food:Groceries,-$30.04', + '7/13/26,0,Payment to xfinity,,Checking (0967),Utilities,-$55.00', + ].join('\n'); + + const { headers, rows } = parseCsv(csv); + const parsed = parseRows({ + rows, + mapping: detectColumns(headers), + dateFormat: detectDateFormat(rows.map((r) => r[0] ?? '')), + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + }); + + expect(parsed.every((p) => !p.error)).toBe(true); + expect(parsed.reduce((sum, p) => sum + p.amount, 0n)).toBe(20328n); + }); +}); + +describe('category column', () => { + const categoryOf = (category: string) => + parseRows({ + rows: [['7/27/26', '0', 'Shop', '', '', category, '-$1.00']], + mapping: MAPPING, + dateFormat: 'M/d/yy', + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + })[0]?.category; + + it('should resolve the categories in the sample export', () => { + expect(categoryOf('Food:Groceries')).toBe('groceries'); + expect(categoryOf('Pets:Pet Supplies')).toBe('pets'); + expect(categoryOf('Utilities')).toBe('utilities'); + expect(categoryOf('Utilities:Telephone/Cellular')).toBe('utilities'); + expect(categoryOf('Utilities:Web Services')).toBe('utilities'); + }); + + it('should fall back to the default for an unknown category', () => { + expect(categoryOf('Widgets')).toBe('general'); + }); + + it('should fall back to the default when the column is blank', () => { + expect(categoryOf('')).toBe('general'); + }); + + it('should fall back to the default when no category column is mapped', () => { + const [parsed] = parseRows({ + rows: [['7/27/26', '0', 'Shop', '', '', 'Food:Groceries', '-$1.00']], + mapping: { ...MAPPING, category: null }, + dateFormat: 'M/d/yy', + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + }); + + expect(parsed?.category).toBe('general'); + }); +}); + +describe('findDuplicateLines', () => { + const existing = [ + { name: 'Whole Foods Market', amount: 11824n, expenseDate: new Date(2026, 6, 27, 13, 30) }, + ]; + + it('should flag a row already present in the group', () => { + const rows = parseAmounts( + [['7/27/26', '0', 'Whole Foods Market', '', '', '', '-$118.24']], + 'expenses_negative', + ); + + expect([...findDuplicateLines(rows, existing)]).toEqual([2]); + }); + + it('should ignore case and surrounding whitespace in the name', () => { + const rows = parseAmounts( + [['7/27/26', '0', ' whole foods market ', '', '', '', '-$118.24']], + 'expenses_negative', + ); + + expect([...findDuplicateLines(rows, existing)]).toEqual([2]); + }); + + it('should not flag a different amount, day or name', () => { + const rows = parseAmounts( + [ + ['7/27/26', '0', 'Whole Foods Market', '', '', '', '-$118.25'], + ['7/28/26', '0', 'Whole Foods Market', '', '', '', '-$118.24'], + ['7/27/26', '0', 'Haggen', '', '', '', '-$118.24'], + ], + 'expenses_negative', + ); + + expect(findDuplicateLines(rows, existing).size).toBe(0); + }); + + it('should ignore rows that already have an error', () => { + const rows = parseAmounts( + [['nope', '0', 'Whole Foods Market', '', '', '', '-$118.24']], + 'expenses_negative', + ); + + expect(findDuplicateLines(rows, existing).size).toBe(0); + }); +}); diff --git a/src/lib/csvImport.ts b/src/lib/csvImport.ts new file mode 100644 index 000000000..8d2c68ee6 --- /dev/null +++ b/src/lib/csvImport.ts @@ -0,0 +1,303 @@ +import { format, isValid, parse, startOfDay } from 'date-fns'; + +import { matchCategory } from '~/lib/category'; +import { getCurrencyHelpers } from '~/utils/numbers'; + +/** + * Which side of zero the source file uses for spending. The opposite sign is money + * received (a refund or reimbursement) and is imported as a negative expense. + */ +export const AMOUNT_SIGNS = ['expenses_negative', 'expenses_positive'] as const; + +export type AmountSign = (typeof AMOUNT_SIGNS)[number]; + +export const isAmountSign = (value: string): value is AmountSign => + (AMOUNT_SIGNS as readonly string[]).includes(value); + +export type CsvColumn = 'date' | 'description' | 'amount' | 'category'; + +/** + * Which CSV column fills each role. Exports often spread the description across several + * fields -- payee, note, memo -- so `description` takes any number of columns and joins + * them in the given order. + */ +export interface ColumnMapping { + date: number | null; + amount: number | null; + category: number | null; + description: number[]; +} + +/** Placed between description columns when more than one contributes a value. */ +export const DESCRIPTION_SEPARATOR = ' - '; + +export type CsvRowError = 'invalid_date' | 'invalid_amount' | 'zero_amount' | 'missing_name'; + +export interface ParsedRow { + /** 1-based line in the source file, counting the header, for error messages. */ + lineNumber: number; + date: Date | null; + name: string; + /** Signed, in the currency's smallest unit. Positive is an expense, negative money received. */ + amount: bigint; + /** A SplitPro category, resolved from the file's own category column where possible. */ + category: string; + raw: Record; + error?: CsvRowError; +} + +/** + * Header names we recognise per role, most specific first. Earlier keywords win over + * later ones, so `Transaction Payee` beats `Note` for the description column. + */ +const COLUMN_KEYWORDS = { + date: ['transaction date', 'date', 'posted', 'day'], + description: [ + 'payee', + 'description', + 'merchant', + 'narrative', + 'details', + 'note', + 'memo', + 'reference', + 'name', + ], + amount: ['amount', 'value', 'debit', 'total', 'sum'], + category: ['category'], +} as const satisfies Record; + +/** + * Date layouts we try to recognise, most common first. Ambiguous values such as + * `7/5/26` match the first entry that fits, which the user can override. + */ +export const DATE_FORMATS = [ + 'M/d/yy', + 'MM/dd/yy', + 'M/d/yyyy', + 'MM/dd/yyyy', + 'd/M/yy', + 'dd/MM/yy', + 'd/M/yyyy', + 'dd/MM/yyyy', + 'yyyy-MM-dd', + 'yyyy/MM/dd', + 'd.M.yyyy', + 'dd.MM.yyyy', + 'd MMM yyyy', + 'dd MMM yyyy', + 'MMM d, yyyy', + 'MMM dd, yyyy', +] as const; + +export type DateFormat = (typeof DATE_FORMATS)[number]; + +export const isDateFormat = (value: string): value is DateFormat => + (DATE_FORMATS as readonly string[]).includes(value); + +/** + * `date-fns` accepts more than the pattern strictly describes -- `yyyy` happily reads a + * two-digit year, for instance. Requiring the parsed date to format back to the original + * text keeps detection honest without hand-written per-format rules. + */ +export const parseDate = (value: string, dateFormat: DateFormat): Date | null => { + const trimmed = value.trim(); + if ('' === trimmed) { + return null; + } + + /* Midnight today: `parse` fills unspecified units from the reference date, and a + real "now" is what resolves a two-digit year to the right century. */ + const parsed = parse(trimmed, dateFormat, startOfDay(new Date())); + + if (!isValid(parsed) || format(parsed, dateFormat) !== trimmed) { + return null; + } + + return parsed; +}; + +/** + * The layout that reads the most samples, preferring the first listed on a tie. Scoring + * rather than requiring a clean sweep means one malformed row cannot derail the whole file. + */ +export const detectDateFormat = (samples: string[]): DateFormat => { + const values = samples.map((s) => s.trim()).filter((s) => '' !== s); + + const best = DATE_FORMATS.reduce<{ dateFormat: DateFormat; matches: number }>( + (acc, dateFormat) => { + const matches = values.filter((value) => parseDate(value, dateFormat)).length; + return matches > acc.matches ? { dateFormat, matches } : acc; + }, + { dateFormat: DATE_FORMATS[0], matches: 0 }, + ); + + return best.dateFormat; +}; + +const scoreHeader = (header: string, keywords: readonly string[]): number => { + const normalized = header.trim().toLowerCase(); + + const keywordIndex = keywords.findIndex((keyword) => normalized.includes(keyword)); + if (-1 === keywordIndex) { + return 0; + } + + const keyword = keywords[keywordIndex]!; + const exactness = normalized === keyword ? 3 : normalized.endsWith(keyword) ? 2 : 1; + + // Keyword priority dominates: a `payee` substring beats an exact `note`. + return (keywords.length - keywordIndex) * 10 + exactness; +}; + +/** Guesses which column holds which value, never assigning one column to two roles. */ +export const detectColumns = (headers: string[]): ColumnMapping => { + const claimed = new Set(); + + /** Every remaining column that matches, in CSV order so the join reads naturally. */ + const pickAll = (keywords: readonly string[]): number[] => { + const matches = headers.reduce((acc, header, index) => { + if (!claimed.has(index) && 0 < scoreHeader(header, keywords)) { + acc.push(index); + } + return acc; + }, []); + + matches.forEach((index) => claimed.add(index)); + return matches; + }; + + const pick = (keywords: readonly string[]): number | null => { + const best = headers.reduce<{ index: number; score: number }>( + (acc, header, index) => { + if (claimed.has(index)) { + return acc; + } + const score = scoreHeader(header, keywords); + return score > acc.score ? { index, score } : acc; + }, + { index: -1, score: 0 }, + ); + + if (0 === best.score) { + return null; + } + + claimed.add(best.index); + return best.index; + }; + + return { + date: pick(COLUMN_KEYWORDS.date), + amount: pick(COLUMN_KEYWORDS.amount), + category: pick(COLUMN_KEYWORDS.category), + description: pickAll(COLUMN_KEYWORDS.description), + }; +}; + +const readCell = (row: string[], index: number | null): string => + null === index ? '' : (row[index] ?? '').trim(); + +/** Joins the mapped description columns, dropping any that are blank on this row. */ +const readDescription = (row: string[], indices: number[]): string => + indices + .map((index) => readCell(row, index)) + .filter((part) => '' !== part) + .join(DESCRIPTION_SEPARATOR); + +const rowError = ({ + raw, + date, + amount, +}: Omit): CsvRowError | undefined => { + if (!date) { + return 'invalid_date'; + } + if (!/\d/.test(raw.amount)) { + return 'invalid_amount'; + } + if (0n === amount) { + return 'zero_amount'; + } + if ('' === raw.description) { + return 'missing_name'; + } + return undefined; +}; + +export interface ParseRowsOptions { + rows: string[][]; + mapping: ColumnMapping; + dateFormat: DateFormat; + amountSign: AmountSign; + currency: string; + locale?: string; +} + +/** + * Turns raw CSV records into rows ready for preview, normalising amounts so that a + * positive value always means an expense regardless of the file's own convention. + */ +export const parseRows = ({ + rows, + mapping, + dateFormat, + amountSign, + currency, + locale, +}: ParseRowsOptions): ParsedRow[] => { + const { toSafeBigInt } = getCurrencyHelpers({ currency, locale }); + const expenseSign = 'expenses_negative' === amountSign ? -1n : 1n; + + return rows.map((row, index) => { + const raw: Record = { + date: readCell(row, mapping.date), + description: readDescription(row, mapping.description), + amount: readCell(row, mapping.amount), + category: readCell(row, mapping.category), + }; + + const date = parseDate(raw.date, dateFormat); + const amount = toSafeBigInt(raw.amount, true) * expenseSign; + + return { + // Offset by the header row and by the zero-based index. + lineNumber: index + 2, + date, + name: raw.description, + amount, + category: matchCategory(raw.category), + raw, + error: rowError({ raw, date, name: raw.description, amount, category: '' }), + }; + }); +}; + +export interface ImportableExpense { + name: string; + amount: bigint; + expenseDate: Date; +} + +const duplicateKey = (name: string, amount: bigint, date: Date) => + `${name.trim().toLowerCase()}|${amount}|${startOfDay(date).getTime()}`; + +/** + * Line numbers of rows that look like a repeat of an expense already in the group -- + * same day, same signed amount and same name. Advisory only; the user decides. + */ +export const findDuplicateLines = ( + rows: ParsedRow[], + existingExpenses: ImportableExpense[], +): Set => { + const existingKeys = new Set( + existingExpenses.map(({ name, amount, expenseDate }) => + duplicateKey(name, amount, expenseDate), + ), + ); + + const isDuplicate = ({ error, date, name, amount }: ParsedRow) => + !error && date && existingKeys.has(duplicateKey(name, amount, date)); + + return new Set(rows.filter(isDuplicate).map((row) => row.lineNumber)); +}; diff --git a/src/pages/groups/[groupId].tsx b/src/pages/groups/[groupId].tsx index e4b824711..e884b487e 100644 --- a/src/pages/groups/[groupId].tsx +++ b/src/pages/groups/[groupId].tsx @@ -5,6 +5,7 @@ import { Check, ChevronLeft, DoorOpen, + FileUp, Info, Merge, PlusIcon, @@ -381,6 +382,13 @@ const BalancePage: NextPageWithUser<{

{t('group_details.group_info.actions')}

+ {!isArchived && ( + + + + )}