diff --git a/README.md b/README.md index ca9887769..010decb32 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ See [docker/README.md](docker/README.md) and [docs/CONFIGURATION.md](docs/CONFIG - Negative expenses are supported for refunds and corrections. - PWA support with push notifications. - Activity feed with edits and deletions. +- Bulk import of a group's expenses from a CSV export, mapping columns and categories. - Detailed balances per person and per group. ## UI preview @@ -52,7 +53,7 @@ Groups are the primary way to use SplitPro. You can invite friends by email, or ### 4) Data utilities -Splitwise import supports friends and groups (partial import). Expenses themselves are not imported yet. You can export data from the balances view and account settings. +Splitwise import supports friends and groups (partial import); Splitwise expenses themselves are not imported yet. A group's expenses can be bulk imported from a CSV export, such as a monthly statement from a bank or a personal finance app. Columns are detected from the header row, the file's own categories are matched onto SplitPro's, and rows carrying the opposite sign are imported as money received. See [docs/CSV_IMPORT.md](docs/CSV_IMPORT.md). You can export data from the balances view and account settings. ### 5) Translations @@ -95,6 +96,7 @@ Bank integration allows you to load transactions from providers like Plaid and c - [docs/CURRENCY_CONVERSIONS.md](docs/CURRENCY_CONVERSIONS.md) - [docs/RECURRING_TRANSACTIONS.md](docs/RECURRING_TRANSACTIONS.md) - [docs/BANK_TRANSACTIONS.md](docs/BANK_TRANSACTIONS.md) +- [docs/CSV_IMPORT.md](docs/CSV_IMPORT.md) - [docker/README.md](docker/README.md) ## Versions diff --git a/docs/CSV_IMPORT.md b/docs/CSV_IMPORT.md new file mode 100644 index 000000000..d46cec362 --- /dev/null +++ b/docs/CSV_IMPORT.md @@ -0,0 +1,146 @@ +# CSV IMPORT + +SplitPro can import a group's expenses in bulk from a CSV file, for example a monthly export from +a bank or a personal finance app. No configuration or external service is required. + +## How to use it + +1. Open a group, then the group info drawer (the ⓘ icon). +2. Under **Actions**, choose **Import expenses from CSV**. +3. Pick the file. SplitPro reads the header row and guesses which columns hold the date, + the description and the amount. +4. Check the options and the preview, then import. + +## What gets created + +Every selected row becomes one expense in the group: + +- Split **equally** between all group members. +- Paid by whoever is selected in **Paid by** (the importing user by default). +- Categorised from the file's own category column where it can be matched, and **General** otherwise. +- Dated from the file's date column. + +Imported expenses are ordinary expenses. Nothing about them is special afterwards, and each one can +be edited or deleted individually. + +## Columns + +Columns are mapped into SplitPro's Date, Description, and Amount. The import operation attempts +to recognize the appropriate columns. + +| Role | Recognised header names | +| ------------------------- | ----------------------------------------------------------------------------------------------- | +| Date | `Transaction Date`, `Date`, `Posted`, `Day` | +| Description (one or more) | `Payee`, `Description`, `Merchant`, `Narrative`, `Details`, `Note`, `Memo`, `Reference`, `Name` | +| Amount | `Amount`, `Value`, `Debit`, `Total`, `Sum` | + +Detection is case-insensitive and matches substrings, so `Transaction Payee` is recognised as the +description. Whatever is detected can be changed, so a file with unrecognised headers still works. + +### Combining columns into the description + +A SplitPro expense has a single description, but exports often spread it over several fields -- +a payee plus a note, or a merchant plus a memo. **Description takes any number of columns**, chosen +with tick boxes rather than a dropdown. Every matching column is ticked automatically, and you can +add or remove any of them. + +Ticked columns are joined in CSV order, separated by `-`. Columns that are blank on a given row +are skipped, so a mostly-empty note column costs nothing: + +| Payee | Note | Imported description | +| -------------------- | ------------- | ---------------------------------- | +| `Whole Foods Market` | `weekly shop` | `Whole Foods Market - weekly shop` | +| `Haggen` | | `Haggen` | + +A row where every ticked column is blank is flagged as missing a description and cannot be +selected. + +## Categories + +SplitPro has a fixed category list, so the file's own categories are matched onto it. The value is +tried whole first, then split on `:`, `.` and `-` and each part tried in turn. A specific item beats +a broad section, and anything unrecognised falls back to **General** rather than failing the row. + +| In the file | Imported as | +| ------------------------ | ---------------------------------------------------- | +| `Food:Groceries` | Groceries | +| `Pets:Pet Supplies` | Pets | +| `Utilities` | Utilities | +| `Utilities:Web Services` | Utilities (the leaf is unknown, the section matches) | +| `Dining-Out` | Dining Out (matched before splitting) | +| `Widgets` | General | + +Matching ignores case, spaces and punctuation, so `Dining Out`, `dining-out` and `diningOut` all +resolve to the same category. The preview shows each row's resolved category as an icon, so a file +whose categories do not match is obvious before you import. + +Leave the category column unmapped to file everything under General. + +## Amount sign and money received + +Exports disagree about which side of zero means spending. The **Amount sign** option says which +convention the file uses: + +- **Expenses are negative** — `-$118.24` is money spent. This is the common bank convention. +- **Expenses are positive** — `118.24` is money spent. + +Rows with the **opposite** sign are money received — a refund, a rebate, a reimbursement — and are +imported as negative expenses, which move balances the other way. They are labelled `Received` in +the preview. + +Getting this option backwards is not silent: every row flips between expense and received in the +preview before anything is written. + +## Dates + +The date layout is detected from the file and can be overridden. `7/27/26` is unambiguously +month-first because 27 cannot be a month; a file where every row is ambiguous (`7/5/26`) defaults to +month-first, so check the preview if your export is day-first. + +Month names are matched in English only. For other languages, export dates in a numeric or ISO +format. + +## Duplicates + +Re-importing a file you have already imported is an easy mistake to make with a monthly workflow. +Rows matching an existing expense in the group on **date, amount, description and currency** are +labelled `Already in group` and start unselected. This is advisory — you can select them anyway. + +## Amount format + +Currency symbols and thousands separators are stripped, so `-$1,234.56` reads correctly. Numbers are +interpreted using the **separators of your app language**: with English selected, `1.234,56` is not +read as 1234.56. The preview shows the parsed value, so a mismatch is visible before importing. + +Any other character makes the amount unreadable, so a typo like `1O.00` is flagged rather than +imported as 1.00. The selected currency's code is allowed, as in `USD 12.00`. + +Accounting-style parentheses for negatives (`(118.24)`) are not recognised as a sign; such a value +reads as positive. Export plain signed numbers instead. + +One currency applies to the whole import; there is no per-row currency column. + +## File format + +Standard RFC 4180 CSV: + +- Comma separated, first line is the header. +- Fields may be quoted; quoted fields may contain commas, newlines and `""`-escaped quotes. +- LF and CRLF line endings are both accepted, as is a leading byte order mark. +- Blank lines are ignored. + +Rows that cannot be used are listed in the preview with a reason (unreadable date, unreadable +amount, zero amount, missing description) and cannot be selected. + +## Example + +```csv +Date,Chk #,Transaction Payee,Note,Account,Category,Amount +7/27/26,0,Whole Foods Market,,Apple Card,Food:Groceries,-$118.24 +7/28/26,0,Haggen,,Apple Card,Food:Groceries,-$30.04 +7/13/26,0,Payment to xfinity,,Checking (0967),Utilities,-$55.00 +8/2/26,0,Utility rebate,,Checking (0967),Utilities,$40.00 +``` + +With **Expenses are negative**, the first three rows import as expenses and the rebate imports as +money received. diff --git a/public/locales/en/common.json b/public/locales/en/common.json index 5ec3f196f..7db7b0af0 100644 --- a/public/locales/en/common.json +++ b/public/locales/en/common.json @@ -312,6 +312,47 @@ "title": "Group statistics", "total_expenses": "Total expenses" }, + "import_csv": { + "title": "Import expenses from CSV", + "note": "Every row becomes an expense split equally between all group members. An attempt to map the incoming category is made and non-matches default to the General category.", + "choose_file": "Choose file", + "no_file_chosen": "No file chosen", + "columns": "Columns", + "column_date": "Date", + "column_description": "Description", + "column_description_hint": "Tick one or more columns. They are joined in order, and blank ones are skipped.", + "column_amount": "Amount", + "column_category": "Category", + "column_unmapped": "Not mapped", + "options": "Options", + "amount_sign": "Amount sign", + "amount_sign_options": { + "expenses_negative": "Expenses are negative", + "expenses_positive": "Expenses are positive" + }, + "date_format": "Date format", + "paid_by": "Paid by", + "currency": "Currency", + "preview": "Preview", + "selected": "{{selected}} of {{total}} selected", + "received": "Received", + "already_in_group": "Already in group", + "no_name": "No description", + "select_row": "Import line {{line}}: {{name}}", + "importing": "Imported {{done}} of {{total}}", + "row_errors": { + "invalid_date": "Unreadable date", + "invalid_amount": "Unreadable amount", + "zero_amount": "Zero amount", + "missing_name": "Missing description" + }, + "messages": { + "import_success": "Imported {{count}} expenses", + "import_partial": "Imported {{count}} expenses before the import failed", + "no_rows_found": "That file has no rows below the header", + "duplicates_unavailable": "Could not load this group's expenses to check for duplicates. Reload to try again." + } + }, "messages": { "default_split_cleared": "Default split cleared", "default_split_updated": "Default split updated", diff --git a/src/components/group/ImportExpensesFromCsv.tsx b/src/components/group/ImportExpensesFromCsv.tsx new file mode 100644 index 000000000..8a063d29e --- /dev/null +++ b/src/components/group/ImportExpensesFromCsv.tsx @@ -0,0 +1,561 @@ +import { PaperClipIcon } from '@heroicons/react/24/solid'; +import { type User } from '@prisma/client'; +import { useRouter } from 'next/router'; +import React, { useCallback, useMemo, useState } from 'react'; +import { toast } from 'sonner'; + +import { useTranslationWithUtils } from '~/hooks/useTranslationWithUtils'; +import { parseCsv } from '~/lib/csv'; +import { + AMOUNT_SIGNS, + type AmountSign, + type ColumnMapping, + type CsvColumn, + DATE_FORMATS, + type DateFormat, + type ParsedRow, + detectColumns, + detectDateFormat, + findDuplicateLines, + isAmountSign, + isDateFormat, + parseRows, +} from '~/lib/csvImport'; +import { type CurrencyCode, parseCurrencyCode } from '~/lib/currency'; +import { cn } from '~/lib/utils'; +import { api } from '~/utils/api'; + +import { CurrencyPicker } from '../AddExpense/CurrencyPicker'; +import { CategoryIcon } from '../ui/categoryIcons'; +import { Button } from '../ui/button'; +import { Checkbox } from '../ui/checkbox'; +import { Input } from '../ui/input'; +import { NativeSelect, NativeSelectOption } from '../ui/native-select'; +import { Separator } from '../ui/separator'; +import { LoadingSpinner } from '../ui/spinner'; +import { buildImportedExpense } from './importExpense'; + +/** Roles filled by exactly one column. Description is chosen separately, and takes several. */ +const SINGLE_COLUMNS = ['date', 'amount', 'category'] as const satisfies readonly CsvColumn[]; + +/** Sentinel for the "not mapped" option, since a select cannot hold null. */ +const UNMAPPED = ''; + +/** Expenses are sent in batches so that a long file is not one oversized request. */ +const IMPORT_BATCH_SIZE = 25; + +const chunk = (items: T[], size: number): T[][] => + items.reduce((acc, item, index) => { + if (0 === index % size) { + acc.push([]); + } + acc[acc.length - 1]!.push(item); + return acc; + }, []); + +export const ImportExpensesFromCsv: React.FC<{ + groupId: number; + user: { id: number; currency?: string | null; defaultCurrency?: string | null }; +}> = ({ groupId, user }) => { + const { t, i18n, displayName, getCurrencyHelpersCached } = useTranslationWithUtils(); + const router = useRouter(); + + const groupDetailQuery = api.group.getGroupDetails.useQuery({ groupId }); + const expensesQuery = api.expense.getGroupExpenses.useQuery({ groupId }); + const addExpenseMutation = api.expense.addOrEditExpense.useMutation(); + + const [fileName, setFileName] = useState(null); + const [headers, setHeaders] = useState([]); + const [records, setRecords] = useState([]); + const [mapping, setMapping] = useState({ + date: null, + amount: null, + category: null, + description: [], + }); + const [dateFormat, setDateFormat] = useState(DATE_FORMATS[0]); + const [amountSign, setAmountSign] = useState('expenses_negative'); + const [currency, setCurrency] = useState( + parseCurrencyCode(user.currency ?? user.defaultCurrency ?? 'USD'), + ); + const [paidById, setPaidById] = useState(user.id); + // Only rows the user has explicitly toggled; everything else follows the default. + const [overrides, setOverrides] = useState>({}); + // Rows an interrupted import already saved; never offered again, so a retry cannot duplicate them. + const [submittedLines, setSubmittedLines] = useState>(new Set()); + const [importedCount, setImportedCount] = useState(0); + /* Tracked separately from the mutation: this stays true across the whole batch run, + where `isPending` would flicker between batches. */ + const [isImporting, setIsImporting] = useState(false); + + // Precomputed so the option keys do not have to be derived from the array index. + const headerOptions = useMemo( + () => headers.map((header, index) => ({ id: `${index}:${header}`, header, index })), + [headers], + ); + + const members = useMemo( + () => groupDetailQuery.data?.groupUsers.map((groupUser) => groupUser.user) ?? [], + [groupDetailQuery.data], + ); + + const rows = useMemo( + () => + parseRows({ + rows: records, + mapping, + dateFormat, + amountSign, + currency, + locale: i18n.language, + }), + [records, mapping, dateFormat, amountSign, currency, i18n.language], + ); + + const duplicateLines = useMemo( + () => + findDuplicateLines( + rows, + (expensesQuery.data ?? []).filter((expense) => expense.currency === currency), + ), + [rows, expensesQuery.data, currency], + ); + + /* Until the group's expenses load, a duplicate looks like a new row, so importing waits. */ + const duplicatesChecked = expensesQuery.isSuccess; + + const isIncluded = useCallback( + (row: ParsedRow) => + !row.error && + !submittedLines.has(row.lineNumber) && + (overrides[row.lineNumber] ?? !duplicateLines.has(row.lineNumber)), + [overrides, duplicateLines, submittedLines], + ); + + const selectedRows = useMemo(() => rows.filter(isIncluded), [rows, isIncluded]); + + const { toUIString } = getCurrencyHelpersCached(currency); + + const onFileChange = useCallback( + async (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + if (!file) { + return; + } + + try { + const parsed = parseCsv(await file.text()); + + if (0 === parsed.rows.length) { + toast.error(t('group_details.import_csv.messages.no_rows_found')); + return; + } + + const detected = detectColumns(parsed.headers); + const dateColumn = detected.date; + + setFileName(file.name); + setHeaders(parsed.headers); + setRecords(parsed.rows); + setMapping(detected); + setOverrides({}); + setSubmittedLines(new Set()); + setImportedCount(0); + setDateFormat( + detectDateFormat( + null === dateColumn ? [] : parsed.rows.map((row) => row[dateColumn] ?? ''), + ), + ); + } catch (error) { + console.error(error); + toast.error(t('errors.import_failed')); + } + }, + [t], + ); + + const onColumnPick = useCallback( + (column: CsvColumn, value: string) => + setMapping((current) => ({ + ...current, + [column]: UNMAPPED === value ? null : Number(value), + })), + [], + ); + + const onDescriptionToggle = useCallback( + (index: number, checked: boolean) => + setMapping((current) => ({ + ...current, + // Kept in CSV order so the joined description reads the way the file is laid out. + description: checked + ? [...current.description, index].sort((a, b) => a - b) + : current.description.filter((i) => i !== index), + })), + [], + ); + + const onCurrencyPick = useCallback((picked: CurrencyCode | null) => { + if (picked) { + setCurrency(picked); + } + }, []); + + const onAmountSignChange = useCallback((event: React.ChangeEvent) => { + if (isAmountSign(event.target.value)) { + setAmountSign(event.target.value); + } + }, []); + + const onDateFormatChange = useCallback((event: React.ChangeEvent) => { + if (isDateFormat(event.target.value)) { + setDateFormat(event.target.value); + } + }, []); + + const onPaidByChange = useCallback( + (event: React.ChangeEvent) => setPaidById(Number(event.target.value)), + [], + ); + + const onRowToggle = useCallback( + (lineNumber: number, included: boolean) => + setOverrides((current) => ({ ...current, [lineNumber]: included })), + [], + ); + + const toExpense = useCallback( + (row: ParsedRow, expenseDate: Date, paidBy: User) => + buildImportedExpense({ row, expenseDate, paidBy, members, currency, groupId }), + [members, currency, groupId], + ); + + const onImport = useCallback(async () => { + const paidBy = members.find((member) => member.id === paidById); + + if (!paidBy) { + return; + } + + const pending = selectedRows.flatMap((row) => + row.date ? [{ lineNumber: row.lineNumber, expense: toExpense(row, row.date, paidBy) }] : [], + ); + + if (0 === pending.length) { + return; + } + + let imported = 0; + const submitted: number[] = []; + setIsImporting(true); + + try { + for (const batch of chunk(pending, IMPORT_BATCH_SIZE)) { + await addExpenseMutation.mutateAsync(batch.map((item) => item.expense)); + imported += batch.length; + submitted.push(...batch.map((item) => item.lineNumber)); + setImportedCount(imported); + } + + toast.success(t('group_details.import_csv.messages.import_success', { count: imported })); + router.push(`/groups/${groupId}`).catch(console.error); + } catch (error) { + console.error(error); + setImportedCount(0); + /* Saved batches stay saved. Deselected only now, not per batch, so the progress total + holds steady while the import runs. */ + setSubmittedLines((current) => new Set([...current, ...submitted])); + toast.error( + 0 === imported + ? t('errors.import_failed') + : t('group_details.import_csv.messages.import_partial', { count: imported }), + ); + } finally { + setIsImporting(false); + } + }, [members, paidById, selectedRows, toExpense, addExpenseMutation, router, groupId, t]); + + const filePicker = ; + + if (!fileName) { + return ( + <> + {filePicker} +

{t('group_details.import_csv.note')}

+ + ); + } + + return ( + <> + {filePicker} + +

{t('group_details.import_csv.columns')}

+
+ {SINGLE_COLUMNS.map((column) => ( + + ))} + +
+

{t('group_details.import_csv.column_description')}

+

+ {t('group_details.import_csv.column_description_hint')} +

+
+ {headerOptions.map((option) => ( + + ))} +
+
+
+ +

{t('group_details.import_csv.options')}

+
+ + + + + + +
+ {t('group_details.import_csv.currency')} +
+ +
+
+
+ +
+

{t('group_details.import_csv.preview')}

+

+ {t('group_details.import_csv.selected', { + selected: selectedRows.length, + total: rows.length, + })} +

+
+ +
+ {rows.map((row) => ( + + ))} +
+ + {expensesQuery.isError ? ( +

+ {t('group_details.import_csv.messages.duplicates_unavailable')} +

+ ) : null} + +
+ +
+ + ); +}; + +interface HeaderOption { + id: string; + header: string; + index: number; +} + +const DescriptionColumn: React.FC<{ + option: HeaderOption; + checked: boolean; + onToggle: (index: number, checked: boolean) => void; +}> = ({ option, checked, onToggle }) => { + const onCheckedChange = useCallback( + (state: boolean | 'indeterminate') => onToggle(option.index, true === state), + [option.index, onToggle], + ); + + return ( + + ); +}; + +const ColumnSelect: React.FC<{ + column: CsvColumn; + options: HeaderOption[]; + value: number | null; + onPick: (column: CsvColumn, value: string) => void; +}> = ({ column, options, value, onPick }) => { + const { t } = useTranslationWithUtils(); + + const onChange = useCallback( + (event: React.ChangeEvent) => onPick(column, event.target.value), + [column, onPick], + ); + + return ( + + ); +}; + +const PreviewRow: React.FC<{ + row: ParsedRow; + included: boolean; + isDuplicate: boolean; + /** False until duplicates can be checked, so a row cannot be picked on incomplete information. */ + selectable: boolean; + amount: string; + onToggle: (lineNumber: number, included: boolean) => void; +}> = ({ row, included, isDuplicate, selectable, amount, onToggle }) => { + const { t, toUIDate } = useTranslationWithUtils(); + + const onCheckedChange = useCallback( + (checked: boolean | 'indeterminate') => onToggle(row.lineNumber, true === checked), + [row.lineNumber, onToggle], + ); + + const label = row.name || t('group_details.import_csv.no_name'); + + const note = [ + row.date ? toUIDate(row.date, { year: true }) : row.raw.date, + row.error && t(`group_details.import_csv.row_errors.${row.error}`), + !row.error && 0n > row.amount && t('group_details.import_csv.received'), + !row.error && isDuplicate && t('group_details.import_csv.already_in_group'), + ] + .filter(Boolean) + .join(' · '); + + return ( + <> +
+
+ + +
+

{label}

+

{note}

+
+
+

row.amount ? 'text-green-500' : '', + )} + > + {amount} +

+
+ + + ); +}; + +const FilePicker: React.FC<{ + fileName: string | null; + onFileChange: (event: React.ChangeEvent) => void; +}> = ({ fileName, onFileChange }) => { + const { t } = useTranslationWithUtils(); + + return ( + + ); +}; diff --git a/src/components/group/importExpense.ts b/src/components/group/importExpense.ts new file mode 100644 index 000000000..a0c946fb9 --- /dev/null +++ b/src/components/group/importExpense.ts @@ -0,0 +1,61 @@ +import { SplitType, type User } from '@prisma/client'; + +import { type ParsedRow } from '~/lib/csvImport'; +import { type SplitShares, calculateParticipantSplit, initSplitShares } from '~/store/addStore'; +import { type CreateExpense } from '~/types/expense.types'; +import { BigMath } from '~/utils/numbers'; + +export interface BuildImportedExpenseOptions { + row: ParsedRow; + expenseDate: Date; + paidBy: User; + members: User[]; + currency: string; + groupId: number; +} + +/** + * Turns one parsed row into an expense split equally across the whole group. + * + * Money received is a negative expense: the magnitude is split as usual and then every + * amount is flipped, which is exactly what AddOrEditExpensePage does for a negative + * amount. The category comes from the row, already resolved to a SplitPro category. + */ +export const buildImportedExpense = ({ + row, + expenseDate, + paidBy, + members, + currency, + groupId, +}: BuildImportedExpenseOptions): CreateExpense => { + const isNegative = 0n > row.amount; + const sign = isNegative ? -1n : 1n; + + const splitShares = members.reduce((acc, member) => { + acc[member.id] = initSplitShares(); + return acc; + }, {}); + + const { participants } = calculateParticipantSplit({ + amount: BigMath.abs(row.amount), + participants: members, + splitType: SplitType.EQUAL, + splitShares, + paidBy, + expenseDate, + isNegative, + }); + + return { + name: row.name, + currency, + amount: row.amount, + groupId, + splitType: SplitType.EQUAL, + paidBy: paidBy.id, + participants: participants.map((p) => ({ userId: p.id, amount: (p.amount ?? 0n) * sign })), + category: row.category, + expenseDate, + }; +}; diff --git a/src/lib/category.test.ts b/src/lib/category.test.ts new file mode 100644 index 000000000..f1ed1ed23 --- /dev/null +++ b/src/lib/category.test.ts @@ -0,0 +1,80 @@ +import { DEFAULT_CATEGORY, matchCategory } from '~/lib/category'; + +describe('matchCategory', () => { + describe('DirectMatches', () => { + it('should match a section by name', () => { + expect(matchCategory('Utilities')).toBe('utilities'); + }); + + it('should match an item by name', () => { + expect(matchCategory('Groceries')).toBe('groceries'); + }); + + it('should ignore case and surrounding whitespace', () => { + expect(matchCategory(' gRoCeRiEs ')).toBe('groceries'); + }); + }); + + describe('NestedCategories', () => { + it('should split on a colon and prefer the specific item', () => { + expect(matchCategory('Food:Groceries')).toBe('groceries'); + }); + + it('should split on a dot', () => { + expect(matchCategory('Food.Groceries')).toBe('groceries'); + }); + + it('should split on a hyphen', () => { + expect(matchCategory('Food-Groceries')).toBe('groceries'); + }); + + it('should fall back to the section when the leaf is unknown', () => { + expect(matchCategory('Utilities:Telephone/Cellular')).toBe('utilities'); + expect(matchCategory('Utilities:Web Services')).toBe('utilities'); + }); + + it('should match an item that appears in a different section', () => { + // `pets` lives under `home`, but the export nests it under its own heading. + expect(matchCategory('Pets:Pet Supplies')).toBe('pets'); + }); + + it('should handle more than two levels', () => { + expect(matchCategory('Expenses:Travel:Hotel')).toBe('hotel'); + }); + }); + + describe('Normalisation', () => { + it('should match a multi-word item written with a space', () => { + expect(matchCategory('Dining Out')).toBe('diningOut'); + }); + + it('should match a multi-word item written with a hyphen', () => { + // The whole string is tried before splitting, so this still resolves. + expect(matchCategory('Dining-Out')).toBe('diningOut'); + }); + + it('should match a multi-word item written in camel case', () => { + expect(matchCategory('diningOut')).toBe('diningOut'); + }); + }); + + describe('Fallback', () => { + it('should fall back for an unknown category', () => { + expect(matchCategory('Widgets')).toBe(DEFAULT_CATEGORY); + }); + + it('should fall back for an empty value', () => { + expect(matchCategory('')).toBe(DEFAULT_CATEGORY); + expect(matchCategory(' ')).toBe(DEFAULT_CATEGORY); + }); + + it('should fall back for separators alone', () => { + expect(matchCategory(':::')).toBe(DEFAULT_CATEGORY); + }); + + it('should never resolve to the shared `other` placeholder', () => { + // `other` is not a storable value -- picking it stores the section name instead. + expect(matchCategory('Other')).toBe(DEFAULT_CATEGORY); + }); + }); +}); diff --git a/src/lib/category.ts b/src/lib/category.ts index 3596b2fbe..8d75fd5cd 100644 --- a/src/lib/category.ts +++ b/src/lib/category.ts @@ -26,3 +26,46 @@ type CategoryValues = (typeof CATEGORIES)[CategorySection][number]; type CategoryWithoutOther = Exclude; export type CategoryItem = CategoryWithoutOther | CategorySection; + +/** Separators used by finance exports to nest categories, e.g. `Food:Groceries`. */ +const CATEGORY_SEPARATORS = /[:.-]/; + +const normalize = (value: string) => value.toLowerCase().replace(/[^a-z0-9]/g, ''); + +const toLookup = (values: readonly string[]) => + new Map(values.map((value) => [normalize(value), value])); + +const SECTION_LOOKUP = toLookup(Object.keys(CATEGORIES)); + +/* `other` is excluded: picking it stores the section name, so it is not a value of its own. */ +const ITEM_LOOKUP = toLookup( + Object.values(CATEGORIES) + .flat() + .filter((item) => 'other' !== item), +); + +/** + * Maps a category from an external system onto a SplitPro category. + * + * The whole string is tried first, then each part after splitting on `:`, `.` and `-`, so + * both `Dining-Out` and `Food:Groceries` resolve. A specific item wins over a section -- + * `Food:Groceries` becomes `groceries`, not `food`. Anything unrecognised falls back to + * the default category rather than failing the row. + */ +export const matchCategory = (value: string): string => { + const candidates = [value, ...value.split(CATEGORY_SEPARATORS)] + .map(normalize) + .filter((candidate) => '' !== candidate); + + const item = candidates.find((candidate) => ITEM_LOOKUP.has(candidate)); + if (item) { + return ITEM_LOOKUP.get(item)!; + } + + const section = candidates.find((candidate) => SECTION_LOOKUP.has(candidate)); + if (section) { + return SECTION_LOOKUP.get(section)!; + } + + return DEFAULT_CATEGORY; +}; diff --git a/src/lib/csv.test.ts b/src/lib/csv.test.ts new file mode 100644 index 000000000..ca52820b2 --- /dev/null +++ b/src/lib/csv.test.ts @@ -0,0 +1,51 @@ +import { parseCsv } from '~/lib/csv'; + +describe('parseCsv', () => { + it('should split a plain record into fields', () => { + const { headers, rows } = parseCsv('a,b,c\n1,2,3\n'); + + expect(headers).toEqual(['a', 'b', 'c']); + expect(rows).toEqual([['1', '2', '3']]); + }); + + it('should keep separators inside quoted fields', () => { + const { rows } = parseCsv('a,b\n"Smith, John",5\n'); + + expect(rows).toEqual([['Smith, John', '5']]); + }); + + it('should unescape doubled quotes', () => { + const { rows } = parseCsv('a\n"He said ""hi"""\n'); + + expect(rows).toEqual([['He said "hi"']]); + }); + + it('should keep newlines inside quoted fields', () => { + const { rows } = parseCsv('a,b\n"line one\nline two",5\n'); + + expect(rows).toEqual([['line one\nline two', '5']]); + }); + + it('should handle CRLF endings and a byte order mark', () => { + const { headers, rows } = parseCsv('\uFEFFa,b\r\n1,2\r\n'); + + expect(headers).toEqual(['a', 'b']); + expect(rows).toEqual([['1', '2']]); + }); + + it('should drop blank lines rather than emit empty records', () => { + const { rows } = parseCsv('a,b\n1,2\n\n\n'); + + expect(rows).toEqual([['1', '2']]); + }); + + it('should preserve empty fields between separators', () => { + const { rows } = parseCsv('a,b,c\n1,,3\n'); + + expect(rows).toEqual([['1', '', '3']]); + }); + + it('should return nothing for empty input', () => { + expect(parseCsv('')).toEqual({ headers: [], rows: [] }); + }); +}); diff --git a/src/lib/csv.ts b/src/lib/csv.ts new file mode 100644 index 000000000..ac534ea1b --- /dev/null +++ b/src/lib/csv.ts @@ -0,0 +1,65 @@ +/** + * Minimal RFC 4180 CSV reader. + * + * Supports quoted fields containing separators, newlines and escaped (`""`) quotes, + * both LF and CRLF line endings, and a leading byte order mark. Fully blank lines + * are dropped, so trailing newlines do not produce empty records. + */ +export const parseCsv = (text: string): { headers: string[]; rows: string[][] } => { + const input = text.startsWith('\uFEFF') ? text.slice(1) : text; + + const records: string[][] = []; + let record: string[] = []; + let field = ''; + let quoted = false; + let index = 0; + + const endField = () => { + record.push(field); + field = ''; + }; + + const endRecord = () => { + endField(); + records.push(record); + record = []; + }; + + while (index < input.length) { + const char = input[index]!; + + if (quoted) { + if ('"' !== char) { + field += char; + index += 1; + } else if ('"' === input[index + 1]) { + // An escaped quote inside a quoted field. + field += '"'; + index += 2; + } else { + quoted = false; + index += 1; + } + } else if ('"' === char && '' === field) { + quoted = true; + index += 1; + } else if (',' === char) { + endField(); + index += 1; + } else if ('\n' === char || '\r' === char) { + endRecord(); + index += '\r' === char && '\n' === input[index + 1] ? 2 : 1; + } else { + field += char; + index += 1; + } + } + + if ('' !== field || 0 < record.length) { + endRecord(); + } + + const [headers = [], ...rows] = records.filter((r) => r.some((f) => '' !== f.trim())); + + return { headers: headers.map((h) => h.trim()), rows }; +}; diff --git a/src/lib/csvImport.test.ts b/src/lib/csvImport.test.ts new file mode 100644 index 000000000..71526d645 --- /dev/null +++ b/src/lib/csvImport.test.ts @@ -0,0 +1,404 @@ +import { parseCsv } from '~/lib/csv'; +import { + type AmountSign, + type ColumnMapping, + DESCRIPTION_SEPARATOR, + detectColumns, + detectDateFormat, + findDuplicateLines, + parseDate, + parseRows, +} from '~/lib/csvImport'; + +const HEADERS = ['Date', 'Chk #', 'Transaction Payee', 'Note', 'Account', 'Category', 'Amount']; + +const MAPPING: ColumnMapping = { date: 0, description: [2], amount: 6, category: 5 }; + +const parseAmounts = (rows: string[][], amountSign: AmountSign) => + parseRows({ + rows, + mapping: MAPPING, + dateFormat: 'M/d/yy', + amountSign, + currency: 'USD', + locale: 'en-US', + }); + +describe('detectColumns', () => { + it('should map the columns of a typical finance export', () => { + // Both `Transaction Payee` and `Note` are description-ish, so both are picked up. + expect(detectColumns(HEADERS)).toEqual({ + date: 0, + description: [2, 3], + amount: 6, + category: 5, + }); + }); + + it('should prefer a payee column over a note column', () => { + expect(detectColumns(['Date', 'Note', 'Payee', 'Amount']).description).toEqual([1, 2]); + }); + + it('should recognise alternative header names', () => { + expect(detectColumns(['Posted', 'Description', 'Value'])).toEqual({ + date: 0, + description: [1], + amount: 2, + category: null, + }); + }); + + it('should never assign one column to two roles', () => { + const mapping = detectColumns(['Amount', 'Amount', 'Amount']); + + expect(mapping.amount).toBe(0); + expect(mapping.date).toBeNull(); + }); + + it('should return nulls when nothing matches', () => { + expect(detectColumns(['foo', 'bar'])).toEqual({ + date: null, + description: [], + amount: null, + category: null, + }); + }); +}); + +describe('description columns', () => { + const parseWith = (description: number[], row: string[]) => + parseRows({ + rows: [row], + mapping: { date: 0, description, amount: 6, category: null }, + dateFormat: 'M/d/yy', + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + })[0]; + + const row = ['7/27/26', '0', 'Whole Foods Market', 'weekly shop', 'Apple Card', 'Food', '-$1.00']; + + it('should use a single column on its own', () => { + expect(parseWith([2], row)?.name).toBe('Whole Foods Market'); + }); + + it('should join several columns in the order given', () => { + expect(parseWith([2, 3], row)?.name).toBe( + `Whole Foods Market${DESCRIPTION_SEPARATOR}weekly shop`, + ); + }); + + it('should respect the caller order rather than sorting', () => { + expect(parseWith([3, 2], row)?.name).toBe( + `weekly shop${DESCRIPTION_SEPARATOR}Whole Foods Market`, + ); + }); + + it('should skip columns that are blank on this row', () => { + const sparse = ['7/27/26', '0', 'Haggen', '', 'Apple Card', 'Food', '-$1.00']; + + expect(parseWith([2, 3], sparse)?.name).toBe('Haggen'); + }); + + it('should join three columns', () => { + expect(parseWith([2, 3, 4], row)?.name).toBe( + ['Whole Foods Market', 'weekly shop', 'Apple Card'].join(DESCRIPTION_SEPARATOR), + ); + }); + + it('should flag a row where every mapped column is blank', () => { + const blank = ['7/27/26', '0', '', '', 'Apple Card', 'Food', '-$1.00']; + + expect(parseWith([2, 3], blank)?.error).toBe('missing_name'); + }); + + it('should flag a row when no column is mapped at all', () => { + expect(parseWith([], row)?.error).toBe('missing_name'); + }); +}); + +describe('parseDate', () => { + it('should read a US short date', () => { + expect(parseDate('7/27/26', 'M/d/yy')).toEqual(new Date(2026, 6, 27)); + }); + + it('should reject a value the format does not describe exactly', () => { + // `date-fns` would otherwise read `26` as the year 26 AD. + expect(parseDate('7/27/26', 'M/d/yyyy')).toBeNull(); + expect(parseDate('07/27/26', 'M/d/yy')).toBeNull(); + }); + + it('should reject an impossible day-month pairing', () => { + expect(parseDate('7/27/26', 'd/M/yy')).toBeNull(); + }); + + it('should reject junk and blanks', () => { + expect(parseDate('not a date', 'M/d/yy')).toBeNull(); + expect(parseDate(' ', 'M/d/yy')).toBeNull(); + }); +}); + +describe('detectDateFormat', () => { + it('should pick month-first when a day exceeds twelve', () => { + expect(detectDateFormat(['7/27/26', '7/4/26'])).toBe('M/d/yy'); + }); + + it('should pick day-first when the first component exceeds twelve', () => { + expect(detectDateFormat(['27/7/26', '4/7/26'])).toBe('d/M/yy'); + }); + + it('should recognise ISO dates', () => { + expect(detectDateFormat(['2026-07-27'])).toBe('yyyy-MM-dd'); + }); + + it('should recognise zero-padded dates', () => { + expect(detectDateFormat(['07/27/2026'])).toBe('MM/dd/yyyy'); + }); + + it('should prefer the layout that fits the most samples', () => { + // The second value rules out month-first, so day-first reads both. + expect(detectDateFormat(['7/4/26', '27/7/26'])).toBe('d/M/yy'); + }); + + it('should tolerate a single unreadable sample', () => { + expect(detectDateFormat(['7/27/26', 'n/a', '7/28/26'])).toBe('M/d/yy'); + }); + + it('should fall back to the most common format when nothing fits', () => { + expect(detectDateFormat(['whenever'])).toBe('M/d/yy'); + }); +}); + +describe('parseRows', () => { + const row = (date: string, payee: string, amount: string) => [ + date, + '0', + payee, + '', + '', + '', + amount, + ]; + + it('should read a negative-is-spending file as expenses', () => { + const [parsed] = parseAmounts( + [row('7/27/26', 'Whole Foods Market', '-$118.24')], + 'expenses_negative', + ); + + expect(parsed).toMatchObject({ + lineNumber: 2, + name: 'Whole Foods Market', + amount: 11824n, + error: undefined, + }); + expect(parsed?.date).toEqual(new Date(2026, 6, 27)); + }); + + it('should read the opposite sign in the same file as money received', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Refund', '$40.00')], 'expenses_negative'); + + expect(parsed?.amount).toBe(-4000n); + }); + + it('should read a positive-is-spending file as expenses', () => { + const [parsed] = parseAmounts( + [row('7/27/26', 'Whole Foods Market', '118.24')], + 'expenses_positive', + ); + + expect(parsed?.amount).toBe(11824n); + }); + + it('should read the opposite sign in a positive-is-spending file as money received', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Refund', '-40.00')], 'expenses_positive'); + + expect(parsed?.amount).toBe(-4000n); + }); + + it('should strip currency symbols and thousands separators', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Rent', '-$1,234.56')], 'expenses_negative'); + + expect(parsed?.amount).toBe(123456n); + }); + + it('should number lines from the source file, header included', () => { + const parsed = parseAmounts( + [row('7/1/26', 'One', '-1.00'), row('7/2/26', 'Two', '-2.00')], + 'expenses_negative', + ); + + expect(parsed.map((p) => p.lineNumber)).toEqual([2, 3]); + }); + + it('should flag an unreadable date', () => { + const [parsed] = parseAmounts([row('nope', 'Shop', '-1.00')], 'expenses_negative'); + + expect(parsed?.error).toBe('invalid_date'); + }); + + it('should flag a missing amount', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Shop', '')], 'expenses_negative'); + + expect(parsed?.error).toBe('invalid_amount'); + }); + + it('should flag an amount with a letter in it instead of dropping the letter', () => { + // `O` typed for `0`: sanitising alone would read this as 1.00. + const [parsed] = parseAmounts([row('7/27/26', 'Shop', '-1O.00')], 'expenses_negative'); + + expect(parsed?.error).toBe('invalid_amount'); + }); + + it('should flag an amount with stray characters in it', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Shop', '-12abc34.56')], 'expenses_negative'); + + expect(parsed?.error).toBe('invalid_amount'); + }); + + it("should accept the selected currency's code", () => { + const [parsed] = parseAmounts([row('7/27/26', 'Shop', 'USD -12.00')], 'expenses_negative'); + + expect(parsed).toMatchObject({ amount: 1200n, error: undefined }); + }); + + it('should accept a currency symbol after the number', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Shop', '-118.24 €')], 'expenses_negative'); + + expect(parsed).toMatchObject({ amount: 11824n, error: undefined }); + }); + + it('should flag a zero amount', () => { + const [parsed] = parseAmounts([row('7/27/26', 'Shop', '$0.00')], 'expenses_negative'); + + expect(parsed?.error).toBe('zero_amount'); + }); + + it('should flag a missing description', () => { + const [parsed] = parseAmounts([row('7/27/26', ' ', '-1.00')], 'expenses_negative'); + + expect(parsed?.error).toBe('missing_name'); + }); + + it('should tolerate an unmapped column', () => { + const [parsed] = parseRows({ + rows: [['7/27/26', '0', 'Shop', '', '', '', '-1.00']], + mapping: { date: 0, description: [], amount: 6, category: null }, + dateFormat: 'M/d/yy', + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + }); + + expect(parsed?.error).toBe('missing_name'); + }); + + it('should parse the sample export end to end', () => { + const csv = [ + 'Date,Chk #,Transaction Payee,Note,Account,Category,Amount', + '7/27/26,0,Whole Foods Market,,Apple Card,Food:Groceries,-$118.24', + '7/28/26,0,Haggen,,Apple Card,Food:Groceries,-$30.04', + '7/13/26,0,Payment to xfinity,,Checking (0967),Utilities,-$55.00', + ].join('\n'); + + const { headers, rows } = parseCsv(csv); + const parsed = parseRows({ + rows, + mapping: detectColumns(headers), + dateFormat: detectDateFormat(rows.map((r) => r[0] ?? '')), + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + }); + + expect(parsed.every((p) => !p.error)).toBe(true); + expect(parsed.reduce((sum, p) => sum + p.amount, 0n)).toBe(20328n); + }); +}); + +describe('category column', () => { + const categoryOf = (category: string) => + parseRows({ + rows: [['7/27/26', '0', 'Shop', '', '', category, '-$1.00']], + mapping: MAPPING, + dateFormat: 'M/d/yy', + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + })[0]?.category; + + it('should resolve the categories in the sample export', () => { + expect(categoryOf('Food:Groceries')).toBe('groceries'); + expect(categoryOf('Pets:Pet Supplies')).toBe('pets'); + expect(categoryOf('Utilities')).toBe('utilities'); + expect(categoryOf('Utilities:Telephone/Cellular')).toBe('utilities'); + expect(categoryOf('Utilities:Web Services')).toBe('utilities'); + }); + + it('should fall back to the default for an unknown category', () => { + expect(categoryOf('Widgets')).toBe('general'); + }); + + it('should fall back to the default when the column is blank', () => { + expect(categoryOf('')).toBe('general'); + }); + + it('should fall back to the default when no category column is mapped', () => { + const [parsed] = parseRows({ + rows: [['7/27/26', '0', 'Shop', '', '', 'Food:Groceries', '-$1.00']], + mapping: { ...MAPPING, category: null }, + dateFormat: 'M/d/yy', + amountSign: 'expenses_negative', + currency: 'USD', + locale: 'en-US', + }); + + expect(parsed?.category).toBe('general'); + }); +}); + +describe('findDuplicateLines', () => { + const existing = [ + { name: 'Whole Foods Market', amount: 11824n, expenseDate: new Date(2026, 6, 27, 13, 30) }, + ]; + + it('should flag a row already present in the group', () => { + const rows = parseAmounts( + [['7/27/26', '0', 'Whole Foods Market', '', '', '', '-$118.24']], + 'expenses_negative', + ); + + expect([...findDuplicateLines(rows, existing)]).toEqual([2]); + }); + + it('should ignore case and surrounding whitespace in the name', () => { + const rows = parseAmounts( + [['7/27/26', '0', ' whole foods market ', '', '', '', '-$118.24']], + 'expenses_negative', + ); + + expect([...findDuplicateLines(rows, existing)]).toEqual([2]); + }); + + it('should not flag a different amount, day or name', () => { + const rows = parseAmounts( + [ + ['7/27/26', '0', 'Whole Foods Market', '', '', '', '-$118.25'], + ['7/28/26', '0', 'Whole Foods Market', '', '', '', '-$118.24'], + ['7/27/26', '0', 'Haggen', '', '', '', '-$118.24'], + ], + 'expenses_negative', + ); + + expect(findDuplicateLines(rows, existing).size).toBe(0); + }); + + it('should ignore rows that already have an error', () => { + const rows = parseAmounts( + [['nope', '0', 'Whole Foods Market', '', '', '', '-$118.24']], + 'expenses_negative', + ); + + expect(findDuplicateLines(rows, existing).size).toBe(0); + }); +}); diff --git a/src/lib/csvImport.ts b/src/lib/csvImport.ts new file mode 100644 index 000000000..c7011829c --- /dev/null +++ b/src/lib/csvImport.ts @@ -0,0 +1,318 @@ +import { format, isValid, parse, startOfDay } from 'date-fns'; + +import { matchCategory } from '~/lib/category'; +import { getCurrencyHelpers } from '~/utils/numbers'; + +/** + * Which side of zero the source file uses for spending. The opposite sign is money + * received (a refund or reimbursement) and is imported as a negative expense. + */ +export const AMOUNT_SIGNS = ['expenses_negative', 'expenses_positive'] as const; + +export type AmountSign = (typeof AMOUNT_SIGNS)[number]; + +export const isAmountSign = (value: string): value is AmountSign => + (AMOUNT_SIGNS as readonly string[]).includes(value); + +export type CsvColumn = 'date' | 'description' | 'amount' | 'category'; + +/** + * Which CSV column fills each role. Exports often spread the description across several + * fields -- payee, note, memo -- so `description` takes any number of columns and joins + * them in the given order. + */ +export interface ColumnMapping { + date: number | null; + amount: number | null; + category: number | null; + description: number[]; +} + +/** Placed between description columns when more than one contributes a value. */ +export const DESCRIPTION_SEPARATOR = ' - '; + +export type CsvRowError = 'invalid_date' | 'invalid_amount' | 'zero_amount' | 'missing_name'; + +export interface ParsedRow { + /** 1-based line in the source file, counting the header, for error messages. */ + lineNumber: number; + date: Date | null; + name: string; + /** Signed, in the currency's smallest unit. Positive is an expense, negative money received. */ + amount: bigint; + /** A SplitPro category, resolved from the file's own category column where possible. */ + category: string; + raw: Record; + error?: CsvRowError; +} + +/** + * Header names we recognise per role, most specific first. Earlier keywords win over + * later ones, so `Transaction Payee` beats `Note` for the description column. + */ +const COLUMN_KEYWORDS = { + date: ['transaction date', 'date', 'posted', 'day'], + description: [ + 'payee', + 'description', + 'merchant', + 'narrative', + 'details', + 'note', + 'memo', + 'reference', + 'name', + ], + amount: ['amount', 'value', 'debit', 'total', 'sum'], + category: ['category'], +} as const satisfies Record; + +/** + * Date layouts we try to recognise, most common first. Ambiguous values such as + * `7/5/26` match the first entry that fits, which the user can override. + */ +export const DATE_FORMATS = [ + 'M/d/yy', + 'MM/dd/yy', + 'M/d/yyyy', + 'MM/dd/yyyy', + 'd/M/yy', + 'dd/MM/yy', + 'd/M/yyyy', + 'dd/MM/yyyy', + 'yyyy-MM-dd', + 'yyyy/MM/dd', + 'd.M.yyyy', + 'dd.MM.yyyy', + 'd MMM yyyy', + 'dd MMM yyyy', + 'MMM d, yyyy', + 'MMM dd, yyyy', +] as const; + +export type DateFormat = (typeof DATE_FORMATS)[number]; + +export const isDateFormat = (value: string): value is DateFormat => + (DATE_FORMATS as readonly string[]).includes(value); + +/** + * `date-fns` accepts more than the pattern strictly describes -- `yyyy` happily reads a + * two-digit year, for instance. Requiring the parsed date to format back to the original + * text keeps detection honest without hand-written per-format rules. + */ +export const parseDate = (value: string, dateFormat: DateFormat): Date | null => { + const trimmed = value.trim(); + if ('' === trimmed) { + return null; + } + + /* Midnight today: `parse` fills unspecified units from the reference date, and a + real "now" is what resolves a two-digit year to the right century. */ + const parsed = parse(trimmed, dateFormat, startOfDay(new Date())); + + if (!isValid(parsed) || format(parsed, dateFormat) !== trimmed) { + return null; + } + + return parsed; +}; + +/** + * The layout that reads the most samples, preferring the first listed on a tie. Scoring + * rather than requiring a clean sweep means one malformed row cannot derail the whole file. + */ +export const detectDateFormat = (samples: string[]): DateFormat => { + const values = samples.map((s) => s.trim()).filter((s) => '' !== s); + + const best = DATE_FORMATS.reduce<{ dateFormat: DateFormat; matches: number }>( + (acc, dateFormat) => { + const matches = values.filter((value) => parseDate(value, dateFormat)).length; + return matches > acc.matches ? { dateFormat, matches } : acc; + }, + { dateFormat: DATE_FORMATS[0], matches: 0 }, + ); + + return best.dateFormat; +}; + +const scoreHeader = (header: string, keywords: readonly string[]): number => { + const normalized = header.trim().toLowerCase(); + + const keywordIndex = keywords.findIndex((keyword) => normalized.includes(keyword)); + if (-1 === keywordIndex) { + return 0; + } + + const keyword = keywords[keywordIndex]!; + const exactness = normalized === keyword ? 3 : normalized.endsWith(keyword) ? 2 : 1; + + // Keyword priority dominates: a `payee` substring beats an exact `note`. + return (keywords.length - keywordIndex) * 10 + exactness; +}; + +/** Guesses which column holds which value, never assigning one column to two roles. */ +export const detectColumns = (headers: string[]): ColumnMapping => { + const claimed = new Set(); + + /** Every remaining column that matches, in CSV order so the join reads naturally. */ + const pickAll = (keywords: readonly string[]): number[] => { + const matches = headers.reduce((acc, header, index) => { + if (!claimed.has(index) && 0 < scoreHeader(header, keywords)) { + acc.push(index); + } + return acc; + }, []); + + matches.forEach((index) => claimed.add(index)); + return matches; + }; + + const pick = (keywords: readonly string[]): number | null => { + const best = headers.reduce<{ index: number; score: number }>( + (acc, header, index) => { + if (claimed.has(index)) { + return acc; + } + const score = scoreHeader(header, keywords); + return score > acc.score ? { index, score } : acc; + }, + { index: -1, score: 0 }, + ); + + if (0 === best.score) { + return null; + } + + claimed.add(best.index); + return best.index; + }; + + return { + date: pick(COLUMN_KEYWORDS.date), + amount: pick(COLUMN_KEYWORDS.amount), + category: pick(COLUMN_KEYWORDS.category), + description: pickAll(COLUMN_KEYWORDS.description), + }; +}; + +const readCell = (row: string[], index: number | null): string => + null === index ? '' : (row[index] ?? '').trim(); + +/** Joins the mapped description columns, dropping any that are blank on this row. */ +const readDescription = (row: string[], indices: number[]): string => + indices + .map((index) => readCell(row, index)) + .filter((part) => '' !== part) + .join(DESCRIPTION_SEPARATOR); + +/** + * Characters a readable amount may contain once the selected currency's code is removed: + * digits, signs, whitespace, separators, parentheses and currency symbols. + */ +const AMOUNT_CHARACTERS = /^[\d\s+.,'()\p{Sc}-]+$/u; + +/** + * Sanitising keeps only digits and separators, so without this check a typo like `1O.00` + * would import as 1.00 instead of being flagged. The selected currency's code is allowed + * because some exports write amounts as `USD 12.00`. + */ +const isReadableAmount = (value: string, currency: string): boolean => { + const withoutCode = value.toUpperCase().split(currency.toUpperCase()).join(''); + return /\d/.test(withoutCode) && AMOUNT_CHARACTERS.test(withoutCode); +}; + +const rowError = ( + { raw, date, amount }: Omit, + currency: string, +): CsvRowError | undefined => { + if (!date) { + return 'invalid_date'; + } + if (!isReadableAmount(raw.amount, currency)) { + return 'invalid_amount'; + } + if (0n === amount) { + return 'zero_amount'; + } + if ('' === raw.description) { + return 'missing_name'; + } + return undefined; +}; + +export interface ParseRowsOptions { + rows: string[][]; + mapping: ColumnMapping; + dateFormat: DateFormat; + amountSign: AmountSign; + currency: string; + locale?: string; +} + +/** + * Turns raw CSV records into rows ready for preview, normalising amounts so that a + * positive value always means an expense regardless of the file's own convention. + */ +export const parseRows = ({ + rows, + mapping, + dateFormat, + amountSign, + currency, + locale, +}: ParseRowsOptions): ParsedRow[] => { + const { toSafeBigInt } = getCurrencyHelpers({ currency, locale }); + const expenseSign = 'expenses_negative' === amountSign ? -1n : 1n; + + return rows.map((row, index) => { + const raw: Record = { + date: readCell(row, mapping.date), + description: readDescription(row, mapping.description), + amount: readCell(row, mapping.amount), + category: readCell(row, mapping.category), + }; + + const date = parseDate(raw.date, dateFormat); + const amount = toSafeBigInt(raw.amount, true) * expenseSign; + + return { + // Offset by the header row and by the zero-based index. + lineNumber: index + 2, + date, + name: raw.description, + amount, + category: matchCategory(raw.category), + raw, + error: rowError({ raw, date, name: raw.description, amount, category: '' }, currency), + }; + }); +}; + +export interface ImportableExpense { + name: string; + amount: bigint; + expenseDate: Date; +} + +const duplicateKey = (name: string, amount: bigint, date: Date) => + `${name.trim().toLowerCase()}|${amount}|${startOfDay(date).getTime()}`; + +/** + * Line numbers of rows that look like a repeat of an expense already in the group -- + * same day, same signed amount and same name. Advisory only; the user decides. + */ +export const findDuplicateLines = ( + rows: ParsedRow[], + existingExpenses: ImportableExpense[], +): Set => { + const existingKeys = new Set( + existingExpenses.map(({ name, amount, expenseDate }) => + duplicateKey(name, amount, expenseDate), + ), + ); + + const isDuplicate = ({ error, date, name, amount }: ParsedRow) => + !error && date && existingKeys.has(duplicateKey(name, amount, date)); + + return new Set(rows.filter(isDuplicate).map((row) => row.lineNumber)); +}; diff --git a/src/pages/groups/[groupId].tsx b/src/pages/groups/[groupId].tsx index e4b824711..5d87380b2 100644 --- a/src/pages/groups/[groupId].tsx +++ b/src/pages/groups/[groupId].tsx @@ -5,6 +5,7 @@ import { Check, ChevronLeft, DoorOpen, + FileUp, Info, Merge, PlusIcon, @@ -381,6 +382,13 @@ const BalancePage: NextPageWithUser<{

{t('group_details.group_info.actions')}

+ {!isArchived && ( + + )}