diff --git a/.github/screenshots/1.png b/.github/screenshots/1.png index cfc87f5..d9bb9c2 100644 Binary files a/.github/screenshots/1.png and b/.github/screenshots/1.png differ diff --git a/.github/screenshots/10.png b/.github/screenshots/10.png index 7e68e3f..ba434b0 100644 Binary files a/.github/screenshots/10.png and b/.github/screenshots/10.png differ diff --git a/.github/screenshots/11.png b/.github/screenshots/11.png index bc5adc5..01214e8 100644 Binary files a/.github/screenshots/11.png and b/.github/screenshots/11.png differ diff --git a/.github/screenshots/12.png b/.github/screenshots/12.png index d7a9c1c..4e8a7fe 100644 Binary files a/.github/screenshots/12.png and b/.github/screenshots/12.png differ diff --git a/.github/screenshots/2.png b/.github/screenshots/2.png index 41fffcd..6abb5dd 100644 Binary files a/.github/screenshots/2.png and b/.github/screenshots/2.png differ diff --git a/.github/screenshots/4.png b/.github/screenshots/4.png index b7ca269..3dade4d 100644 Binary files a/.github/screenshots/4.png and b/.github/screenshots/4.png differ diff --git a/.github/screenshots/5.png b/.github/screenshots/5.png index 18daa3a..636c7c3 100644 Binary files a/.github/screenshots/5.png and b/.github/screenshots/5.png differ diff --git a/.github/screenshots/6.png b/.github/screenshots/6.png index ca77a47..d2e9aa5 100644 Binary files a/.github/screenshots/6.png and b/.github/screenshots/6.png differ diff --git a/.github/screenshots/7.png b/.github/screenshots/7.png index d0f394c..eb60139 100644 Binary files a/.github/screenshots/7.png and b/.github/screenshots/7.png differ diff --git a/.github/screenshots/8.png b/.github/screenshots/8.png index c80f61b..91b3c3a 100644 Binary files a/.github/screenshots/8.png and b/.github/screenshots/8.png differ diff --git a/README.md b/README.md index 1e5aa54..6b33ee1 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,9 @@ Explorer or a table's right-click menu. editable before you commit to it, and named in your engine's own dialect - Headers optional, `NULL` text configurable (`\N`, `NULL`, whatever your exporter writes), leading lines skippable for files with a preamble, whitespace trimmable +- **Empty fields keep their meaning**: a bare one is `NULL`, a quoted `""` is an empty string - + the same convention Postgres `COPY … FORMAT csv` uses, and what lets a XenSQL CSV export load + straight back in. In a numeric or date column, where `""` is never valid, both mean `NULL` - UTF-8 BOMs stripped, ragged rows tolerated - **Bad rows are skipped and reported**, not fatal - or tick **stop at the first error** if you'd rather nothing partial lands @@ -320,6 +323,8 @@ transactions behave. You get a progress bar and a count - not thousands of resul - Export all or **selected** rows and columns - Copy to clipboard or **save to file** - Remembers your last export format +- **CSV and SQL exports import back cleanly** - `NULL` stays apart from an empty string, and cells + that spreadsheets would treat as formulas are guarded on the way out and unguarded on the way in --- diff --git a/build/config.yml b/build/config.yml index 8586bb1..7a9689a 100644 --- a/build/config.yml +++ b/build/config.yml @@ -11,7 +11,7 @@ info: description: "A fast, native SQL client built with Go, Wails and React." # The application description copyright: "(c) 2026, Bare7a" # Copyright text comments: "A fast, native SQL client built with Go, Wails and React." # Comments - version: "1.5.0" # The application version + version: "1.5.1" # The application version # cfBundleIconName: "appicon" # The macOS icon name in Assets.car icon bundles (optional) # # Should match the name of your .icon file without the extension # # If not set and Assets.car exists, defaults to "appicon" diff --git a/build/darwin/Info.dev.plist b/build/darwin/Info.dev.plist index 68b27bc..9af4f6f 100644 --- a/build/darwin/Info.dev.plist +++ b/build/darwin/Info.dev.plist @@ -78,9 +78,9 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.5.0 + 1.5.1 CFBundleVersion - 1.5.0 + 1.5.1 LSMinimumSystemVersion 12.0.0 NSAppTransportSecurity diff --git a/build/darwin/Info.plist b/build/darwin/Info.plist index 2a53fcb..39599fa 100644 --- a/build/darwin/Info.plist +++ b/build/darwin/Info.plist @@ -78,9 +78,9 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.5.0 + 1.5.1 CFBundleVersion - 1.5.0 + 1.5.1 LSMinimumSystemVersion 12.0.0 NSHighResolutionCapable diff --git a/build/ios/Info.dev.plist b/build/ios/Info.dev.plist index 5652400..26da2e1 100644 --- a/build/ios/Info.dev.plist +++ b/build/ios/Info.dev.plist @@ -15,9 +15,9 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.5.0-dev + 1.5.1-dev CFBundleVersion - 1.5.0 + 1.5.1 LSRequiresIPhoneOS MinimumOSVersion diff --git a/build/ios/Info.plist b/build/ios/Info.plist index d873b5f..8bad98c 100644 --- a/build/ios/Info.plist +++ b/build/ios/Info.plist @@ -15,9 +15,9 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.5.0 + 1.5.1 CFBundleVersion - 1.5.0 + 1.5.1 LSRequiresIPhoneOS MinimumOSVersion diff --git a/build/linux/nfpm/nfpm.yaml b/build/linux/nfpm/nfpm.yaml index d39fc28..2e2e263 100644 --- a/build/linux/nfpm/nfpm.yaml +++ b/build/linux/nfpm/nfpm.yaml @@ -6,7 +6,7 @@ name: "XenSQL" arch: ${GOARCH} platform: "linux" -version: "1.5.0" +version: "1.5.1" section: "default" priority: "extra" maintainer: ${GIT_COMMITTER_NAME} <${GIT_COMMITTER_EMAIL}> diff --git a/build/windows/info.json b/build/windows/info.json index ec262c9..542ebde 100644 --- a/build/windows/info.json +++ b/build/windows/info.json @@ -1,10 +1,10 @@ { "fixed": { - "file_version": "1.5.0" + "file_version": "1.5.1" }, "info": { "0000": { - "ProductVersion": "1.5.0", + "ProductVersion": "1.5.1", "CompanyName": "Bare7a", "FileDescription": "A fast, native SQL client built with Go, Wails and React.", "LegalCopyright": "(c) 2026, Bare7a", diff --git a/build/windows/nsis/wails_tools.nsh b/build/windows/nsis/wails_tools.nsh index e22a758..944b926 100644 --- a/build/windows/nsis/wails_tools.nsh +++ b/build/windows/nsis/wails_tools.nsh @@ -14,7 +14,7 @@ !define INFO_PRODUCTNAME "XenSQL" !endif !ifndef INFO_PRODUCTVERSION - !define INFO_PRODUCTVERSION "1.5.0" + !define INFO_PRODUCTVERSION "1.5.1" !endif !ifndef INFO_COPYRIGHT !define INFO_COPYRIGHT "(c) 2026, Bare7a" diff --git a/build/windows/wails.exe.manifest b/build/windows/wails.exe.manifest index 8551ffd..95a13d7 100644 --- a/build/windows/wails.exe.manifest +++ b/build/windows/wails.exe.manifest @@ -1,6 +1,6 @@ - + diff --git a/e2e/pages/app-page.ts b/e2e/pages/app-page.ts index 4728773..5409ace 100644 --- a/e2e/pages/app-page.ts +++ b/e2e/pages/app-page.ts @@ -52,10 +52,21 @@ export class AppPage { // ── Test isolation ─────────────────────────────────────────────────────── /** Reset shared backend state: the editor session (tabs) and saved connections both persist and restore on load. */ async resetState(): Promise { + await this.dismissModals(); await this.closeAllTabs(); await this.resetConnections(); } + /** A modal left open by a test covers the whole page and would block every click below. */ + async dismissModals(): Promise { + for (let i = 0; i < 3; i++) { + const overlay = this.page.locator('.modal-overlay').last(); + if (!(await overlay.isVisible().catch(() => false))) return; + await this.page.keyboard.press('Escape'); + await overlay.waitFor({ state: 'hidden', timeout: 1_000 }).catch(() => {}); + } + } + /** * Close every editor tab, clearing the persisted session so the next test's page loads clean. * Runs at teardown, when this test's tabs are already rendered (nothing is restoring), so a diff --git a/e2e/pages/results-page.ts b/e2e/pages/results-page.ts index d097d4b..f3a90dd 100644 --- a/e2e/pages/results-page.ts +++ b/e2e/pages/results-page.ts @@ -151,4 +151,11 @@ export class ResultsPage { await this.exportDialog.getByRole('button', { name: 'Cancel' }).click(); await expect(this.exportDialog).toBeHidden(); } + + /** Copies the export to the clipboard (which closes the dialog) and returns its text. */ + async copyExportToClipboard(): Promise { + await this.exportDialog.getByRole('button', { name: 'Copy to clipboard' }).click(); + await expect(this.exportDialog).toBeHidden(); + return this.page.evaluate(() => navigator.clipboard.readText()); + } } diff --git a/e2e/screenshots/readme.spec.ts b/e2e/screenshots/readme.spec.ts index a032d34..d387c5c 100644 --- a/e2e/screenshots/readme.spec.ts +++ b/e2e/screenshots/readme.spec.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import type { Page } from '@playwright/test'; import { expect, test } from '@support/fixtures'; +import { stubImportFilePicker } from '@support/importFile'; import type { SchemaPage } from '../pages/schema-page'; import { COLOR, COLOR_INDEX, FORUM, POSTGRES_DEV, POSTGRES_READONLY } from './screenshot-db'; @@ -13,9 +14,6 @@ const OUT_DIR = path.resolve(screenshotsDir, '../../.github/screenshots'); const IMPORT_CSV = process.env.XENSQL_SCREENSHOT_CSV ?? path.join(screenshotsDir, 'fixtures', 'users.csv'); const IMPORT_CSV_COLUMNS = 13; -/** Wails binding ID of `App.PickImportFile` (frontend/bindings/…/app.ts). */ -const PICK_IMPORT_FILE_ID = 452154835; - const GET_ALL_POSTS_SQL = 'SELECT * FROM posts p ORDER BY p.id;'; const PLAN_SQL = `SELECT c.name AS category, u.display_name AS author, COUNT(*) AS posts, SUM(p.views) AS views @@ -218,15 +216,6 @@ async function waitForCellViewerSyntax(page: Page): Promise { await page.waitForTimeout(250); } -/** Server mode has no native file dialog, so answer that one binding with a fixture path. */ -async function stubImportFilePicker(page: Page, filePath: string): Promise { - await page.route('**/wails/runtime', async (route) => { - const body = route.request().postDataJSON() as { args?: { methodID?: number } } | null; - if (body?.args?.methodID !== PICK_IMPORT_FILE_ID) return route.fallback(); - await route.fulfill({ contentType: 'application/json', body: JSON.stringify(filePath) }); - }); -} - async function createFolder(page: Page, name: string): Promise { // Assumes the connection switcher menu is already open. await page.locator('.sidebar-connections-toolbar button[data-tooltip="New folder"]').click(); diff --git a/e2e/specs/sidebar/import.spec.ts b/e2e/specs/sidebar/import.spec.ts index e807194..f87b46a 100644 --- a/e2e/specs/sidebar/import.spec.ts +++ b/e2e/specs/sidebar/import.spec.ts @@ -1,8 +1,11 @@ +import fs from 'node:fs'; import { POSTGRES } from '@support/databases'; import { expect, test } from '@support/fixtures'; +import { stubImportFilePicker, writeCSVFixture } from '@support/importFile'; -// The native file picker can't be driven here, so these cover the dialog only; the load itself is -// covered by internal/app/app_import_test.go. +test.use({ permissions: ['clipboard-read', 'clipboard-write'] }); + +// The native picker is stubbed at the binding; row-level behaviour lives in app_import_test.go. test.describe('Import dialog', () => { test('opens from the schema toolbar with CSV selected', async ({ connections, page }) => { await connections.createAndConnect(POSTGRES); @@ -54,4 +57,82 @@ test.describe('Import dialog', () => { await expect(dialog).toBeVisible(); await expect(dialog.getByLabel('Table')).toHaveValue(table); }); + + test('loads a CSV into an existing table, counting rows against the file total', async ({ + connections, + page, + schema, + seed, + }) => { + await connections.createAndConnect(POSTGRES); + const table = await seed.table('e2e_import_run'); + await schema.refresh(); + + const rows = ['id,name']; + for (let i = 1; i <= 1200; i++) rows.push(`${i},name${i}`); + const csv = writeCSVFixture('people.csv', rows); + await stubImportFilePicker(page, csv); + + await schema.openTableMenu(table); + await page.getByRole('menuitem', { name: 'Import into this table…' }).click(); + const dialog = page.getByRole('dialog'); + await dialog.getByRole('button', { name: 'Browse' }).click(); + await expect(dialog.locator('#import-path')).toHaveValue(csv); + await expect(dialog.locator('.import-preview-head')).toContainText('1,200 row(s)'); + await page.unroute('**/wails/runtime'); + + await dialog.getByRole('button', { name: 'Import', exact: true }).click(); + await expect(dialog.locator('.task-progress')).toContainText(/Importing… [\d,]+ of 1,200 rows read/); + // Reaching a result proves import:done landed; without it the button stays on "Importing…". + await expect(dialog.locator('.import-result')).toContainText('Imported 1,200 row(s), skipped 0', { + timeout: 30_000, + }); + await expect(dialog.getByRole('button', { name: 'Import', exact: true })).toBeEnabled(); + }); + + test('re-imports a CSV the export dialog produced, NULLs and formula-shaped text intact', async ({ + connections, + editor, + page, + results, + schema, + seed, + }) => { + await connections.createAndConnect(POSTGRES); + const table = await seed.table('e2e_roundtrip', { columns: '(id INT, txt TEXT, note TEXT)' }); + await editor.run( + `INSERT INTO ${table} (id, txt, note) VALUES (1, NULL, 'plain'), (2, '', '-not a number'), (3, 'x', '=1+2');`, + ); + await schema.refresh(); + + await editor.run(`SELECT id, txt, note FROM ${table} ORDER BY id;`); + await results.waitForRows(); + await results.openExportDialog(); + await results.setExportFormat('csv'); + const csv = (await results.copyExportToClipboard()).replace(/\r\n/g, '\n'); + expect(csv).toBe(`id,txt,note\n1,,plain\n2,"",'-not a number\n3,x,'=1+2`); + + const file = writeCSVFixture('roundtrip.csv', csv.split('\n')); + await stubImportFilePicker(page, file); + await schema.openTableMenu(table); + await page.getByRole('menuitem', { name: 'Import into this table…' }).click(); + const dialog = page.getByRole('dialog'); + await dialog.getByRole('button', { name: 'Browse' }).click(); + await expect(dialog.locator('#import-path')).toHaveValue(file); + await page.unroute('**/wails/runtime'); + await dialog.getByRole('button', { name: 'Import', exact: true }).click(); + await expect(dialog.locator('.import-result')).toContainText('Imported 3 row(s), skipped 0', { timeout: 30_000 }); + await dialog.locator('.modal-footer').getByRole('button', { name: 'Close', exact: true }).click(); + fs.rmSync(file, { force: true }); + + // 6 = the 3 originals plus 3 identical re-imports: NULL, '' and '=1+2' all intact. + await editor.run( + `SELECT COUNT(*) AS intact FROM ${table} + WHERE (id = 1 AND txt IS NULL AND note = 'plain') + OR (id = 2 AND txt = '' AND note = '-not a number') + OR (id = 3 AND txt = 'x' AND note = '=1+2');`, + ); + await results.waitForRows(); + await expect(results.grid).toContainText('6'); + }); }); diff --git a/e2e/support/importFile.ts b/e2e/support/importFile.ts new file mode 100644 index 0000000..66fc05b --- /dev/null +++ b/e2e/support/importFile.ts @@ -0,0 +1,24 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { Page } from '@playwright/test'; + +/** Wails binding ID of `App.PickImportFile` (frontend/bindings/…/app.ts). */ +const PICK_IMPORT_FILE_ID = 452154835; + +/** Answers the PickImportFile binding with a fixture path; unroute once the dialog has it. */ +export async function stubImportFilePicker(page: Page, filePath: string): Promise { + await page.route('**/wails/runtime', async (route) => { + const body = route.request().postDataJSON() as { args?: { methodID?: number } } | null; + if (body?.args?.methodID !== PICK_IMPORT_FILE_ID) return route.fallback(); + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(filePath) }); + }); +} + +/** Writes a CSV to a temp dir and returns its path. */ +export function writeCSVFixture(name: string, lines: string[]): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'xensql-e2e-')); + const file = path.join(dir, name); + fs.writeFileSync(file, `${lines.join('\n')}\n`); + return file; +} diff --git a/frontend/bindings/xensql/internal/app/models.ts b/frontend/bindings/xensql/internal/app/models.ts index 79781e1..0343270 100644 --- a/frontend/bindings/xensql/internal/app/models.ts +++ b/frontend/bindings/xensql/internal/app/models.ts @@ -152,6 +152,11 @@ export class ImportPreview { "totalBytes": number; "truncated": boolean; + /** + * 0 when the file was too large to count. + */ + "totalRows": number; + /** Creates a new ImportPreview instance. */ constructor($$source: Partial = {}) { if (!("columns" in $$source)) { @@ -175,6 +180,9 @@ export class ImportPreview { if (!("truncated" in $$source)) { this["truncated"] = false; } + if (!("totalRows" in $$source)) { + this["totalRows"] = 0; + } Object.assign(this, $$source); } diff --git a/frontend/bindings/xensql/internal/service/models.ts b/frontend/bindings/xensql/internal/service/models.ts index 867e2df..2e27956 100644 --- a/frontend/bindings/xensql/internal/service/models.ts +++ b/frontend/bindings/xensql/internal/service/models.ts @@ -13,7 +13,7 @@ export class CSVOptions { "hasHeader": boolean; /** - * NullLiteral is the text that becomes SQL NULL; blank fields always are. + * NullLiteral is extra text that becomes SQL NULL; a bare empty field already is. */ "nullLiteral"?: string; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c8d33a4..e894036 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "xensql-frontend", - "version": "1.5.0", + "version": "1.5.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xensql-frontend", - "version": "1.5.0", + "version": "1.5.1", "dependencies": { "@fontsource/fira-code": "^5.3.0", "@fontsource/inter": "^5.3.0", @@ -14,7 +14,7 @@ "@tanstack/react-virtual": "^3.14.9", "@wailsio/runtime": "latest", "i18next": "^26.3.6", - "lucide-react": "^1.28.0", + "lucide-react": "^1.31.0", "monaco-editor": "^0.56.0", "react": "^19.2.8", "react-dom": "^19.2.8", @@ -23,13 +23,13 @@ "zustand": "^5.0.14" }, "devDependencies": { - "@biomejs/biome": "2.5.6", - "@types/node": "^26.1.2", + "@biomejs/biome": "2.5.7", + "@types/node": "^26.2.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", "@vitejs/plugin-react": "^6.0.5", "typescript": "^7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", "vitest": "^4.1.10" } }, @@ -43,9 +43,9 @@ } }, "node_modules/@biomejs/biome": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.6.tgz", - "integrity": "sha512-lxVNjv7UF6KfhMJfL9gaUHbWdJdHbsAj6OSmwSYNdhRuG67NxNQ4Xdvh3TUxsSK9sBzJBQhEJj3AopmmNJ5pSA==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.7.tgz", + "integrity": "sha512-zr8K/DcY5tYsQOQwqMJ0AWElo6QgmgNI7idXgXLhevVszlt8RGVpesEJPqx3ThazLaOwjJ5Y8fz3BtH5fGZNsw==", "dev": true, "license": "MIT OR Apache-2.0", "bin": { @@ -59,20 +59,20 @@ "url": "https://opencollective.com/biome" }, "optionalDependencies": { - "@biomejs/cli-darwin-arm64": "2.5.6", - "@biomejs/cli-darwin-x64": "2.5.6", - "@biomejs/cli-linux-arm64": "2.5.6", - "@biomejs/cli-linux-arm64-musl": "2.5.6", - "@biomejs/cli-linux-x64": "2.5.6", - "@biomejs/cli-linux-x64-musl": "2.5.6", - "@biomejs/cli-win32-arm64": "2.5.6", - "@biomejs/cli-win32-x64": "2.5.6" + "@biomejs/cli-darwin-arm64": "2.5.7", + "@biomejs/cli-darwin-x64": "2.5.7", + "@biomejs/cli-linux-arm64": "2.5.7", + "@biomejs/cli-linux-arm64-musl": "2.5.7", + "@biomejs/cli-linux-x64": "2.5.7", + "@biomejs/cli-linux-x64-musl": "2.5.7", + "@biomejs/cli-win32-arm64": "2.5.7", + "@biomejs/cli-win32-x64": "2.5.7" } }, "node_modules/@biomejs/cli-darwin-arm64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.6.tgz", - "integrity": "sha512-zMOLZP4oMrjh6m1zcSj1ud2awUPgTuMVbmQhYYWL7J8HwCnbHHBvTm7VBTRuY7epT5bez76IpKYQ11ZAqHFlnw==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.7.tgz", + "integrity": "sha512-vxo/Ls3/PYdQWyLhYYcgMOCzQypAjcY+iihS8M0wW03l16TCLW4zqZzGo75gm1VdCMj38hTVZ31KBWrZ4G9dJw==", "cpu": [ "arm64" ], @@ -87,9 +87,9 @@ } }, "node_modules/@biomejs/cli-darwin-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.6.tgz", - "integrity": "sha512-JAC1VqzvO7Th5ZplU0G2uGfkZbxEe9uDDektPAhF0JLusoz1w+T4okp2bkykI0bbaO2vslKiRfj4gU43JaGreA==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.7.tgz", + "integrity": "sha512-Cd3Ga61amT/Yl/0x8elP5hhGYaFy4bw6WuysTgf7oo8TA5tJ5A1k+DkVoJ2BHbTVil51gTX9VPzArnrlLJ3Kyg==", "cpu": [ "x64" ], @@ -104,9 +104,9 @@ } }, "node_modules/@biomejs/cli-linux-arm64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.6.tgz", - "integrity": "sha512-6XsYwCFkp5sMxl85ffhgeGpGgs6A7dRYFnkceZ7WVxvycuTnGdD5xa534Z3xfrBQ0JCMK/mujT6ZNPJoghedwg==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.7.tgz", + "integrity": "sha512-rR2QE0yF2GYSuYuKIa7pKvODGJqnOH+2eDREAM8wV+mWKSkMQKdAp4zXEZfTaxY8PMoNONnpgSWcBCyLDPDOKg==", "cpu": [ "arm64" ], @@ -124,9 +124,9 @@ } }, "node_modules/@biomejs/cli-linux-arm64-musl": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.6.tgz", - "integrity": "sha512-eUa3jeeYvfMt19LBeh6E5PUZpxnTC4JqNWo+EDjTtQjAr2xLGnWaxACtVU1DQqmHYbvThlJzLX+ZsYgrqh2qVw==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.7.tgz", + "integrity": "sha512-xPI5yB6XlpDbNkS+bm1t42olw5c4l3UrlOmLg7KtLJvjvkNF/1V4tnUgfkylGIeb3u/T+BzMGYqgQhzjAoJzuQ==", "cpu": [ "arm64" ], @@ -144,9 +144,9 @@ } }, "node_modules/@biomejs/cli-linux-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.6.tgz", - "integrity": "sha512-Pop9VXCFUhFTMfFefZ39S+u2rOPyNp5iHlxbZRwXGACHLy2r0jjiRgJHmaEKJzL3SyxlVeGShXhvvElvWowonA==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.7.tgz", + "integrity": "sha512-FQgqJhscrqJUFptGaRSUJWlXAExwWcDwLuK49dvKfkQ1bB5SEEyFssnsxQY83Xm6jR0EbbX3+8+D5bfvYqUG2Q==", "cpu": [ "x64" ], @@ -164,9 +164,9 @@ } }, "node_modules/@biomejs/cli-linux-x64-musl": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.6.tgz", - "integrity": "sha512-2Vp13QdKysH3HIWLaYLhUUwbK+jbZonJD1K+Lr0d0RO4wH7mkYd43vJixEDm8cUWrowoRz4UUHF1nm9Ae7ym8A==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.7.tgz", + "integrity": "sha512-rE5VZi+qtmPgQH+l7jVxYoZ18b/TiHEhulhMpjmCZH1PltSbjRcxNWywC3HZ9tYottG7ORkeTtoscBilKSBm0g==", "cpu": [ "x64" ], @@ -184,9 +184,9 @@ } }, "node_modules/@biomejs/cli-win32-arm64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.6.tgz", - "integrity": "sha512-tDGshcm6BdkZOCGnTDX0Y8/U4IfBSlnUU7T56nNDuPEfed+aHg+u8G36NB43fJVl0Os6+QURXIE1yuD7AaEofA==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.7.tgz", + "integrity": "sha512-Oq4x0CCwP4jirrcTywXs5kOGZ4v5vuEP+gWrbtjApOA2CL9F3F9GlIdQIci8AKSCa/zURanMRpX/4wQ7Am6hHg==", "cpu": [ "arm64" ], @@ -201,9 +201,9 @@ } }, "node_modules/@biomejs/cli-win32-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.6.tgz", - "integrity": "sha512-WN05KwXnTO/2J45RQPvzZMXf7tZUIofHoR35xIPfCo7pQ2RFidxI8sfb5mGsaTxdMmEOzHzOPRCdA5/fCpc7xQ==", + "version": "2.5.7", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.7.tgz", + "integrity": "sha512-V+0wu/nrj2S+MhP4EQ0uHNolP0IALEsz45pg0WoKkHfDeh0+ItHwP/p7bX5RPoMOl9NkpHYWdYPhIcy2mACHvQ==", "cpu": [ "x64" ], @@ -681,9 +681,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.1.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.2.tgz", - "integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==", + "version": "26.2.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", + "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", "dev": true, "license": "MIT", "dependencies": { @@ -1245,9 +1245,9 @@ "license": "MIT" }, "node_modules/dompurify": { - "version": "3.4.12", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.12.tgz", - "integrity": "sha512-zQvGet8Z2sWbQhCmfFz/T5QWH2oBmjnqK3qvOjaqaNLrLEF912WamU+ohnTp0TCep/MFVHpdJuCZEdFOdTnEFg==", + "version": "3.4.13", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", + "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -1624,9 +1624,9 @@ } }, "node_modules/lucide-react": { - "version": "1.28.0", - "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.28.0.tgz", - "integrity": "sha512-fARAFJULsGuDDydjp6+6blekG/sBIM29TerzLjc9bQUKAcEfrSc4ZQKb25KRz4OMKd87cZTb5dgq0w/T6KufVg==", + "version": "1.31.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.31.0.tgz", + "integrity": "sha512-G8u2eEtoHUnUa9f8lbvqDhCiORMnYLdUEo06EEG9MQvHQrInKcX3Pa2TH39MM5qyzRcWETxB0+aOwAPI1g1kEg==", "license": "ISC", "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" @@ -1671,9 +1671,9 @@ "license": "BSD-3-Clause" }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -2048,16 +2048,16 @@ } }, "node_modules/vite": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.0.tgz", - "integrity": "sha512-pn+CFpM0lwDeKwmOq1ZaBK/9sjorZcgqxki6MbY/jPEVd9vichIlmlD4HmQ5wdP5EgqQCFRaACBxMC7uEGc6lQ==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.1.tgz", + "integrity": "sha512-EU/eS7BH3XROHh2YnBefjM6DBKA6ZeMZEYQbj7NLWg5wHYlhB8B/Mayd5XsgWq+NFYccDOTemRpdETWR6Ka/lw==", "dev": true, "license": "MIT", "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.5", - "postcss": "^8.5.23", - "rolldown": "~1.2.0", + "postcss": "^8.5.25", + "rolldown": "~1.2.1", "tinyglobby": "^0.2.17" }, "bin": { diff --git a/frontend/package.json b/frontend/package.json index 455ae81..3c5ad38 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "xensql-frontend", "private": true, - "version": "1.5.0", + "version": "1.5.1", "homepage": "https://xensql.bare7a.eu", "type": "module", "scripts": { @@ -24,7 +24,7 @@ "@tanstack/react-virtual": "^3.14.9", "@wailsio/runtime": "latest", "i18next": "^26.3.6", - "lucide-react": "^1.28.0", + "lucide-react": "^1.31.0", "monaco-editor": "^0.56.0", "react": "^19.2.8", "react-dom": "^19.2.8", @@ -33,16 +33,16 @@ "zustand": "^5.0.14" }, "devDependencies": { - "@biomejs/biome": "2.5.6", - "@types/node": "^26.1.2", + "@biomejs/biome": "2.5.7", + "@types/node": "^26.2.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", "@vitejs/plugin-react": "^6.0.5", "typescript": "^7.0.2", - "vite": "^8.2.0", + "vite": "^8.2.1", "vitest": "^4.1.10" }, "overrides": { - "dompurify": "^3.4.12" + "dompurify": "^3.4.13" } } diff --git a/frontend/src/features/import/ImportDialog.tsx b/frontend/src/features/import/ImportDialog.tsx index 825cd00..746fc12 100644 --- a/frontend/src/features/import/ImportDialog.tsx +++ b/frontend/src/features/import/ImportDialog.tsx @@ -1,8 +1,10 @@ +import type { TFunction } from 'i18next'; import { CircleAlert, CircleCheck, FileSpreadsheet, FileText, FolderOpen, Loader2 } from 'lucide-react'; import { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { useImportRun } from '@/features/import/hooks/useImportRun'; +import { type ImportProgress, useImportRun } from '@/features/import/hooks/useImportRun'; import { Modal } from '@/shared/components/Modal'; +import { formatCount, TaskProgress } from '@/shared/components/TaskProgress'; import { api } from '@/shared/lib/api'; import { cx } from '@/shared/lib/cx'; import { formatError } from '@/shared/lib/normalize'; @@ -107,6 +109,15 @@ export function ImportDialog({ connectionId, schema, tables, initialTable, onClo } }; + const shownProgress: ImportProgress = progress ?? { + processed: 0, + inserted: 0, + skipped: 0, + bytesRead: 0, + totalBytes: preview?.totalBytes ?? 0, + totalRows: kind === 'csv' ? (preview?.totalRows ?? 0) : 0, + }; + const targetTable = target === 'new' ? newTable.trim() : existingTable; const mappedCount = mapping.filter((m) => m.trim()).length; const canRun = @@ -167,7 +178,7 @@ export function ImportDialog({ connectionId, schema, tables, initialTable, onClo setColumnTypes((prev) => prev.map((c, idx) => (idx === i ? value : c))); return ( - +
@@ -373,6 +384,7 @@ export function ImportDialog({ connectionId, schema, tables, initialTable, onClo
{t('import.columnsHeading')} + {preview.totalRows > 0 && `${t('import.fileRows', { rows: formatCount(preview.totalRows) })} · `} {t('import.detectedDelimiter', { delimiter: displayDelimiter(preview.delimiter) })}
@@ -429,19 +441,8 @@ export function ImportDialog({ connectionId, schema, tables, initialTable, onClo {kind !== 'csv' && stopOnErrorCheck} - {running && progress && ( -
-

- {kind === 'sql' - ? t('import.progressStatements', { done: progress.processed, total: progress.totalBytes }) - : t('import.progressRows', { rows: progress.processed, inserted: progress.inserted })} -

- -
+ {running && ( + )} {error && ( @@ -455,8 +456,11 @@ export function ImportDialog({ connectionId, schema, tables, initialTable, onClo

{result.cancelled ? : }{' '} {kind === 'sql' - ? t('import.doneStatements', { count: result.statements, skipped: result.skipped }) - : t('import.doneRows', { count: result.inserted, skipped: result.skipped })} + ? t('import.doneStatements', { + count: formatCount(result.statements), + skipped: formatCount(result.skipped), + }) + : t('import.doneRows', { count: formatCount(result.inserted), skipped: formatCount(result.skipped) })}

{result.cancelled &&

{t('import.cancelledNote')}

} {!!result.errors?.length && ( @@ -485,3 +489,22 @@ export function ImportDialog({ connectionId, schema, tables, initialTable, onClo function displayDelimiter(d: string): string { return d === '\t' ? '\\t' : d; } + +function progressLabel(t: TFunction, kind: ImportKind, p: ImportProgress): string { + if (kind === 'sql') { + return t('import.progressStatements', { done: formatCount(p.processed), total: formatCount(p.totalBytes) }); + } + if (p.totalRows > 0) { + return t('import.progressRowsOf', { + done: formatCount(Math.min(p.processed, p.totalRows)), + total: formatCount(p.totalRows), + }); + } + return t('import.progressRows', { rows: formatCount(p.processed) }); +} + +function progressBar(kind: ImportKind, p: ImportProgress): { value: number; max: number } { + if (kind === 'sql') return { value: p.processed, max: p.totalBytes }; + if (p.totalRows > 0) return { value: Math.min(p.processed, p.totalRows), max: p.totalRows }; + return { value: p.bytesRead, max: p.totalBytes }; +} diff --git a/frontend/src/features/import/hooks/useImportRun.ts b/frontend/src/features/import/hooks/useImportRun.ts index dbe84fc..ed4713e 100644 --- a/frontend/src/features/import/hooks/useImportRun.ts +++ b/frontend/src/features/import/hooks/useImportRun.ts @@ -16,6 +16,7 @@ export interface ImportProgress { skipped: number; bytesRead: number; totalBytes: number; + totalRows: number; } // The frontend mints the id, so a cancelled run's events can't be mistaken for this one's. @@ -28,7 +29,7 @@ export function useImportRun(connectionId: string | null) { useEffect(() => { const unsubProgress = Events.On('import:progress', (e) => { - const p = (e.data as ImportProgressPayload[] | undefined)?.[0]; + const p = e.data as ImportProgressPayload | undefined; if (!p || p.importId !== activeId.current) return; setProgress({ processed: p.processed, @@ -36,10 +37,11 @@ export function useImportRun(connectionId: string | null) { skipped: p.skipped, bytesRead: p.bytesRead, totalBytes: p.totalBytes, + totalRows: p.totalRows, }); }); const unsubDone = Events.On('import:done', (e) => { - const d = (e.data as ImportDonePayload[] | undefined)?.[0]; + const d = e.data as ImportDonePayload | undefined; if (!d || d.importId !== activeId.current) return; activeId.current = null; setRunning(false); diff --git a/frontend/src/i18n/locales/bg.json b/frontend/src/i18n/locales/bg.json index 7f2690f..e17fdb0 100644 --- a/frontend/src/i18n/locales/bg.json +++ b/frontend/src/i18n/locales/bg.json @@ -696,6 +696,7 @@ "reading": "Четене на файла", "columnsHeading": "Колони", "detectedDelimiter": "Разделител: {{delimiter}}", + "fileRows": "{{rows}} ред(а)", "sourceColumn": "Във файла", "sample": "Първа стойност", "targetColumn": "Зареди в", @@ -706,8 +707,9 @@ "run": "Импортирай", "running": "Импортиране…", "stop": "Спри", - "progressRows": "{{rows}} прочетен(и) ред(а) · {{inserted}} вмъкнати", - "progressStatements": "Заявка {{done}} от {{total}}", + "progressRows": "Импортиране… {{rows}} реда прочетени", + "progressRowsOf": "Импортиране… {{done}} от {{total}} реда прочетени", + "progressStatements": "Импортиране… {{done}} от {{total}} заявки изпълнени", "doneRows": "Импортирани {{count}} ред(а), пропуснати {{skipped}}", "doneStatements": "Изпълнени {{count}} заявка(и), пропуснати {{skipped}}", "cancelledNote": "Спряно предсрочно - вече заредените редове са запазени.", diff --git a/frontend/src/i18n/locales/de.json b/frontend/src/i18n/locales/de.json index 4c92fa8..8fec50a 100644 --- a/frontend/src/i18n/locales/de.json +++ b/frontend/src/i18n/locales/de.json @@ -314,7 +314,7 @@ "formatJson": "JSON", "formatMarkdown": "Markdown", "formatSql": "SQL INSERT", - "progress": "Exportiert… {{done}} von {{total}} Zeilen geschrieben", + "progress": "Exportiere… {{done}} von {{total}} Zeilen geschrieben", "stop": "Stoppen" }, "sidebar": { @@ -696,6 +696,7 @@ "reading": "Datei wird gelesen", "columnsHeading": "Spalten", "detectedDelimiter": "Trennzeichen: {{delimiter}}", + "fileRows": "{{rows}} Zeile(n)", "sourceColumn": "In der Datei", "sample": "Erster Wert", "targetColumn": "Laden nach", @@ -706,8 +707,9 @@ "run": "Importieren", "running": "Importiere…", "stop": "Anhalten", - "progressRows": "{{rows}} Zeile(n) gelesen · {{inserted}} eingefügt", - "progressStatements": "Anweisung {{done}} von {{total}}", + "progressRows": "Importiere… {{rows}} Zeilen gelesen", + "progressRowsOf": "Importiere… {{done}} von {{total}} Zeilen gelesen", + "progressStatements": "Importiere… {{done}} von {{total}} Anweisungen ausgeführt", "doneRows": "{{count}} Zeile(n) importiert, {{skipped}} übersprungen", "doneStatements": "{{count}} Anweisung(en) ausgeführt, {{skipped}} übersprungen", "cancelledNote": "Vorzeitig angehalten - die bereits geladenen Zeilen bleiben erhalten.", diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 87aa589..f47b87f 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -696,6 +696,7 @@ "reading": "Reading the file", "columnsHeading": "Columns", "detectedDelimiter": "Delimiter: {{delimiter}}", + "fileRows": "{{rows}} row(s)", "sourceColumn": "In the file", "sample": "First value", "targetColumn": "Load into", @@ -706,8 +707,9 @@ "run": "Import", "running": "Importing…", "stop": "Stop", - "progressRows": "{{rows}} row(s) read · {{inserted}} inserted", - "progressStatements": "Statement {{done}} of {{total}}", + "progressRows": "Importing… {{rows}} rows read", + "progressRowsOf": "Importing… {{done}} of {{total}} rows read", + "progressStatements": "Importing… {{done}} of {{total}} statements run", "doneRows": "Imported {{count}} row(s), skipped {{skipped}}", "doneStatements": "Ran {{count}} statement(s), skipped {{skipped}}", "cancelledNote": "Stopped early - the rows already loaded were kept.", diff --git a/frontend/src/shared/components/ExportResultsDialog.tsx b/frontend/src/shared/components/ExportResultsDialog.tsx index 221adf9..bf451ff 100644 --- a/frontend/src/shared/components/ExportResultsDialog.tsx +++ b/frontend/src/shared/components/ExportResultsDialog.tsx @@ -1,6 +1,7 @@ import { useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Modal } from '@/shared/components/Modal'; +import { formatCount, TaskProgress } from '@/shared/components/TaskProgress'; import { api } from '@/shared/lib/api'; import { appToast, toastError } from '@/shared/lib/appToast'; import { writeChunkedFile } from '@/shared/lib/exportFile'; @@ -203,12 +204,14 @@ export function ExportResultsDialog({ })}

) : ( -
-

- {t('export.progress', { done: progress, total: exportOptions.rowIndices.length })} -

- -
+ )}
diff --git a/frontend/src/shared/components/TaskProgress.tsx b/frontend/src/shared/components/TaskProgress.tsx new file mode 100644 index 0000000..bee1ab4 --- /dev/null +++ b/frontend/src/shared/components/TaskProgress.tsx @@ -0,0 +1,19 @@ +interface Props { + label: string; + value: number; + max: number; +} + +/** The progress line shared by the export and import dialogs. */ +export function TaskProgress({ label, value, max }: Props) { + return ( +
+

{label}

+ +
+ ); +} + +export function formatCount(n: number): string { + return n.toLocaleString(); +} diff --git a/frontend/src/shared/lib/appInfo.ts b/frontend/src/shared/lib/appInfo.ts index 20ce895..cb93efc 100644 --- a/frontend/src/shared/lib/appInfo.ts +++ b/frontend/src/shared/lib/appInfo.ts @@ -11,7 +11,7 @@ export interface AppInfo { /** Fallback when Go binding is unavailable (dev in browser). */ export const DEFAULT_APP_INFO: AppInfo = { name: 'XenSQL', - version: '1.5.0', + version: '1.5.1', author: 'Bare7a', email: 'bare7a@gmail.com', website: 'https://xensql.bare7a.eu', diff --git a/frontend/src/shared/lib/exportResult.test.ts b/frontend/src/shared/lib/exportResult.test.ts index 9252872..b244a92 100644 --- a/frontend/src/shared/lib/exportResult.test.ts +++ b/frontend/src/shared/lib/exportResult.test.ts @@ -80,6 +80,18 @@ describe('exportResultToText - csv', () => { const out = exportResultToText(sample(), 'csv'); expect(out).toContain('3,eve,\n'.trimEnd()); }); + it('writes NULL bare but quotes an empty string, so the two survive a re-import', () => { + const r: QueryResult = { + columns: ['a', 'b'], + columnTypes: ['text', 'text'], + rows: [[null, '']], + rowCount: 1, + affectedRows: 0, + durationMs: 0, + tableName: 't', + }; + expect(exportResultToText(r, 'csv').split('\n')[1]).toBe(',""'); + }); it('quotes leading-whitespace fields and the \\. sentinel (matching Go csv)', () => { const r: QueryResult = { columns: ['v'], diff --git a/frontend/src/shared/lib/exportResult.ts b/frontend/src/shared/lib/exportResult.ts index 7dfafec..651f3ef 100644 --- a/frontend/src/shared/lib/exportResult.ts +++ b/frontend/src/shared/lib/exportResult.ts @@ -106,8 +106,8 @@ function jsonFormatter(view: ExportView): RowFormatter { function csvFormatter(view: ExportView): RowFormatter { const escapeCsv = (cell: string) => { - // Mirror Go's encoding/csv: quote on delimiter/quote/newline, leading whitespace, or \. sentinel. - const needsQuote = cell === '\\.' || /[",\n\r]/.test(cell) || /^\s/u.test(cell); + // Mirrors Go's escaping; '' is quoted too, telling it apart from NULL on re-import. + const needsQuote = cell === '' || cell === '\\.' || /[",\n\r]/.test(cell) || /^\s/u.test(cell); if (needsQuote) return `"${cell.replace(/"/g, '""')}"`; return cell; }; diff --git a/frontend/src/styles/import.css b/frontend/src/styles/import.css index 633c6bb..2918b3f 100644 --- a/frontend/src/styles/import.css +++ b/frontend/src/styles/import.css @@ -7,7 +7,9 @@ margin: 0; } +/* flex-shrink: 0 or the flex-column body squashes this box and overflow:hidden clips the rows. */ .import-preview { + flex-shrink: 0; border: 1px solid var(--border); border-radius: var(--radius); overflow: hidden; @@ -24,11 +26,6 @@ font-size: var(--text-sm); } -.import-map-scroll { - max-height: 21.5rem; - overflow: auto; -} - /* Fixed layout so the columns split the dialog evenly in every language instead of hugging the widest source name. */ .import-map-table { @@ -46,9 +43,6 @@ } .import-map-table thead th { - position: sticky; - top: 0; - z-index: 1; background: var(--bg-elevated); font-weight: 600; font-size: var(--text-2xs); @@ -61,10 +55,11 @@ border-top: 1px solid var(--border); } +/* A width floor would push the fixed-layout table sideways. */ .import-map-table input, .import-map-table select { width: 100%; - min-width: 90px; + min-width: 0; } .import-source-name { @@ -89,12 +84,6 @@ opacity: 0.45; } -.import-progress { - display: flex; - flex-direction: column; - gap: var(--space-5); -} - .import-result { display: flex; flex-direction: column; diff --git a/frontend/src/styles/results.css b/frontend/src/styles/results.css index 5e65c4a..3dfd1d7 100644 --- a/frontend/src/styles/results.css +++ b/frontend/src/styles/results.css @@ -301,34 +301,6 @@ margin: var(--space-4) 0 0; } -.export-progress-bar { - /* Reset the native widget so the track/fill colours below apply. */ - appearance: none; - display: block; - width: 100%; - height: 0.25rem; - margin-top: var(--space-6); - border: none; - border-radius: var(--radius-pill); - background: var(--bg-hover); - color: var(--accent); -} - -.export-progress-bar::-webkit-progress-bar { - background: var(--bg-hover); - border-radius: var(--radius-pill); -} - -.export-progress-bar::-webkit-progress-value { - background: var(--accent); - border-radius: var(--radius-pill); -} - -.export-progress-bar::-moz-progress-bar { - background: var(--accent); - border-radius: var(--radius-pill); -} - .results-table-wrap { flex: 1; min-height: 0; diff --git a/frontend/src/styles/utilities.css b/frontend/src/styles/utilities.css index 4fce7b2..ddcfa8c 100644 --- a/frontend/src/styles/utilities.css +++ b/frontend/src/styles/utilities.css @@ -155,3 +155,38 @@ .app-empty-flex { flex: 1; } + +/* TaskProgress (export and import dialogs). */ +.task-progress-label { + margin: var(--space-4) 0 0; + font-size: var(--text-sm); + color: var(--text-muted); +} + +.task-progress-bar { + /* Reset the native widget so the track/fill colours below apply. */ + appearance: none; + display: block; + width: 100%; + height: 0.25rem; + margin-top: var(--space-6); + border: none; + border-radius: var(--radius-pill); + background: var(--bg-hover); + color: var(--accent); +} + +.task-progress-bar::-webkit-progress-bar { + background: var(--bg-hover); + border-radius: var(--radius-pill); +} + +.task-progress-bar::-webkit-progress-value { + background: var(--accent); + border-radius: var(--radius-pill); +} + +.task-progress-bar::-moz-progress-bar { + background: var(--accent); + border-radius: var(--radius-pill); +} diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index f27e24b..816a287 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -143,6 +143,8 @@ export interface ImportPreview { delimiter: string; totalBytes: number; truncated: boolean; + /** 0 when the file was too large to count. */ + totalRows: number; } export interface CSVImportRequest { @@ -182,6 +184,8 @@ export interface ImportProgressPayload { skipped: number; bytesRead: number; totalBytes: number; + /** 0 when unknown; bytesRead then drives the bar. */ + totalRows: number; } export interface ImportDonePayload { diff --git a/go.mod b/go.mod index cac781d..83d420e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/go-sql-driver/mysql v1.10.0 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.10.0 - github.com/wailsapp/wails/v3 v3.0.0-beta.4 + github.com/wailsapp/wails/v3 v3.0.0-beta.6 golang.org/x/crypto v0.54.0 modernc.org/sqlite v1.56.0 ) @@ -30,7 +30,7 @@ require ( golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.40.0 // indirect - modernc.org/libc v1.74.4 // indirect + modernc.org/libc v1.75.3 // indirect modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect + modernc.org/memory v1.12.0 // indirect ) diff --git a/go.sum b/go.sum index 63a69ef..6a9daf3 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/wailsapp/wails/v3 v3.0.0-beta.4 h1:Kv5ywwZDMB0SgA1zhLM4fImf5ZsGWATTkoAQ3DS/pB8= -github.com/wailsapp/wails/v3 v3.0.0-beta.4/go.mod h1:BzATbK71VFikMMMCo434wAi0QcaI03P+xeaWgDvQvjw= +github.com/wailsapp/wails/v3 v3.0.0-beta.6 h1:k9FHF/T39EyTZNCHweRrLt1c6dwV3R3A+r1oCNiPB8I= +github.com/wailsapp/wails/v3 v3.0.0-beta.6/go.mod h1:A/OaL1mXOnwWynTJv4rZU89Wbk5q3rtq3C3qkx4rRN0= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= @@ -68,30 +68,30 @@ golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= -modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= -modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= -modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= -modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/libc v1.75.3 h1:vCqT5+R0jPXMnvMkGo0T2zXvFNth+lYXVCx5X7CCX/g= +modernc.org/libc v1.75.3/go.mod h1:MjAX68G+0oufI+hNuh0QXcK+Ap+sL8bNPPcIC6EqOfo= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/memory v1.12.0 h1:twkmYNkGXCvtYWzoux02jtK6eovjZbdI0uHFUYp6kuU= +modernc.org/memory v1.12.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= diff --git a/internal/app/app_import.go b/internal/app/app_import.go index f5df99f..7422957 100644 --- a/internal/app/app_import.go +++ b/internal/app/app_import.go @@ -2,13 +2,13 @@ package app import ( "context" - "encoding/csv" "errors" "fmt" "io" "os" "strings" "sync/atomic" + "time" "xensql/internal/database" "xensql/internal/service" @@ -21,6 +21,11 @@ const maxReportedErrors = 20 const defaultImportBatchSize = 500 +// Above this the row count is skipped; bytes drive the bar. +const maxRowCountBytes = 64 << 20 + +const progressInterval = 100 * time.Millisecond + type ImportPreview struct { Columns []string `json:"columns"` Rows [][]string `json:"rows"` @@ -31,6 +36,8 @@ type ImportPreview struct { Delimiter string `json:"delimiter"` TotalBytes int64 `json:"totalBytes"` Truncated bool `json:"truncated"` + // 0 when the file was too large to count. + TotalRows int64 `json:"totalRows"` } type CSVImportRequest struct { @@ -72,6 +79,8 @@ type ImportProgressEvent struct { Skipped int64 `json:"skipped"` BytesRead int64 `json:"bytesRead"` TotalBytes int64 `json:"totalBytes"` + // 0 when unknown; BytesRead then drives the bar. + TotalRows int64 `json:"totalRows"` } type ImportDoneEvent struct { @@ -120,6 +129,8 @@ func (a *App) PreviewImportFile(connectionID, path string, opts service.CSVOptio if info, statErr := file.Stat(); statErr == nil { totalBytes = info.Size() } + // Before the sample, which consumes the reader. + totalRows := countCSVRows(file, opts, totalBytes) reader, err := service.NewCSVReader(file, opts) if err != nil { return ImportPreview{}, err @@ -152,13 +163,46 @@ func (a *App) PreviewImportFile(connectionID, path string, opts service.CSVOptio Rows: rows, InferredTypes: inferred, SQLTypes: sqlTypes, - Delimiter: string(reader.Comma), + Delimiter: string(reader.Comma()), TotalBytes: totalBytes, Truncated: truncated, + TotalRows: totalRows, }, nil } -func readSample(reader *csv.Reader, hasHeader bool, limit int) (columns []string, rows [][]string, truncated bool, err error) { +// countCSVRows counts with the import's own reader settings; 0 when too big to scan twice. +func countCSVRows(file *os.File, opts service.CSVOptions, totalBytes int64) int64 { + defer func() { _, _ = file.Seek(0, io.SeekStart) }() + if totalBytes > maxRowCountBytes { + return 0 + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return 0 + } + reader, err := service.NewCSVReader(file, opts) + if err != nil { + return 0 + } + if opts.HasHeader { + if _, hErr := reader.Read(); hErr != nil { + return 0 + } + } + var rows int64 + for { + _, readErr := reader.Read() + if readErr == io.EOF { + return rows + } + if readErr != nil { + // An I/O failure must not report a half count. + return 0 + } + rows++ + } +} + +func readSample(reader *service.CSVReader, hasHeader bool, limit int) (columns []string, rows [][]string, truncated bool, err error) { first, err := reader.Read() if err == io.EOF { return nil, nil, false, fmt.Errorf("the file is empty") @@ -167,10 +211,10 @@ func readSample(reader *csv.Reader, hasHeader bool, limit int) (columns []string return nil, nil, false, err } if hasHeader { - columns = service.UniqueColumnNames(first) + columns = service.UniqueColumnNames(service.CSVValues(first)) } else { columns = service.PositionalHeader(len(first)) - rows = append(rows, padRow(first, len(columns))) + rows = append(rows, padRow(service.CSVValues(first), len(columns))) } for len(rows) < limit { rec, readErr := reader.Read() @@ -181,7 +225,7 @@ func readSample(reader *csv.Reader, hasHeader bool, limit int) (columns []string // A malformed row shouldn't sink the preview; show what we have. return columns, rows, false, nil } - rows = append(rows, padRow(rec, len(columns))) + rows = append(rows, padRow(service.CSVValues(rec), len(columns))) } if _, readErr := reader.Read(); readErr == nil { truncated = true @@ -256,6 +300,8 @@ func (a *App) runCSVImport( if info, statErr := file.Stat(); statErr == nil { totalBytes = info.Size() } + // Before the byte counter, so bytesRead covers only the import's own pass. + em.totalRows = countCSVRows(file, req.Options, totalBytes) counter := &countingReader{r: file} reader, err := service.NewCSVReader(counter, req.Options) if err != nil { @@ -294,7 +340,7 @@ func (a *App) runCSVImport( } } - boolCols := boolColumns(req.ColumnTypes, sources) + conv := a.valueConverter(ctx, s, schema, req, targets, sources) result := &ImportResult{} batch := make([][]any, 0, batchSize) @@ -332,20 +378,21 @@ func (a *App) runCSVImport( continue } processed++ - batch = append(batch, buildValues(record, sources, boolCols, req.Options.NullLiteral, req.Options.TrimSpace)) + batch = append(batch, conv.buildValues(record, sources)) if len(batch) >= batchSize { if fErr := flush(); fErr != nil { return nil, fErr } - em.progress(processed, result.Inserted, result.Skipped, counter.n.Load(), totalBytes) } + // Throttled, so a file smaller than one batch still reports progress. + em.progress(processed, result.Inserted, result.Skipped, counter.n.Load(), totalBytes) } if !result.Cancelled { if fErr := flush(); fErr != nil { return nil, fErr } } - em.progress(processed, result.Inserted, result.Skipped, counter.n.Load(), totalBytes) + em.progressNow(processed, result.Inserted, result.Skipped, counter.n.Load(), totalBytes) return result, nil } @@ -382,36 +429,162 @@ func execInsert(ctx context.Context, s database.Session, schema, table string, t return s.ExecuteArgs(ctx, stmt, args) } -func buildValues(record []string, sources []int, boolCols map[int]bool, nullLiteral string, trim bool) []any { +// valueConv turns one CSV record into insert parameters; built once per run. +type valueConv struct { + // boolCols hit real boolean columns; boolIntCols load bool-shaped data into numerics as 1/0. + boolCols map[int]bool + boolIntCols map[int]bool + // dateCols need MySQL's datetime literal rather than RFC 3339. + dateCols map[int]bool + // textCols can hold ''; elsewhere a quoted empty field still means NULL. + textCols map[int]bool + nullLiteral string + trim bool + // pgx encodes only Go bools into Postgres booleans; MySQL/SQLite take 1/0. + boolAsBool bool +} + +// A quoted empty field is ”; a bare one is NULL - the distinction Postgres COPY csv draws. +func (c *valueConv) buildValues(record []service.CSVField, sources []int) []any { out := make([]any, len(sources)) for i, src := range sources { if src >= len(record) { out[i] = nil continue } - v := record[src] - if trim { + field := record[src] + v := field.Value + if c.trim { v = strings.TrimSpace(v) } - if v == "" || (nullLiteral != "" && v == nullLiteral) { + if c.nullLiteral != "" && v == c.nullLiteral { out[i] = nil continue } - if boolCols[i] { - out[i] = normalizeBool(v) + if v == "" { + if field.Quoted && c.textCols[i] { + out[i] = "" + continue + } + out[i] = nil continue } - out[i] = v + v = undefuseFormula(v) + switch { + case c.boolCols[i]: + out[i] = normalizeBool(v, c.boolAsBool) + case c.boolIntCols[i]: + out[i] = normalizeBool(v, false) + case c.dateCols[i]: + out[i] = mysqlDateTime(v) + default: + out[i] = v + } } return out } +// Reverses the export's spreadsheet-formula guard (a leading ' before = + - @). +func undefuseFormula(v string) string { + if len(v) < 2 || v[0] != '\'' { + return v + } + switch v[1] { + case '=', '+', '-', '@': + return v[1:] + } + return v +} + +// Rewrites RFC 3339 into the literal MySQL accepts; anything else passes through. +func mysqlDateTime(v string) string { + if ts, err := time.Parse(time.RFC3339Nano, v); err == nil { + return ts.Format("2006-01-02 15:04:05.999999") + } + return v +} + +// New tables: types from the request; existing tables: the catalog. +func (a *App) valueConverter( + ctx context.Context, + s database.Session, + schema string, + req CSVImportRequest, + targets []string, + sources []int, +) *valueConv { + conv := &valueConv{ + boolCols: map[int]bool{}, + boolIntCols: map[int]bool{}, + dateCols: map[int]bool{}, + textCols: map[int]bool{}, + nullLiteral: req.Options.NullLiteral, + trim: req.Options.TrimSpace, + boolAsBool: s.DriverType() == database.DriverPostgres, + } + mysql := s.DriverType() == database.DriverMySQL + reqBool := boolColumns(req.ColumnTypes, sources) + + if req.CreateTable { + conv.boolCols = reqBool + for i, src := range sources { + t := database.ImportText + if src < len(req.ColumnTypes) && req.ColumnTypes[src] != "" { + t = database.ImportColumnType(req.ColumnTypes[src]) + } + conv.textCols[i] = t == database.ImportText + if mysql && (t == database.ImportDate || t == database.ImportTimestamp) { + conv.dateCols[i] = true + } + } + return conv + } + + cols, err := s.ListColumns(ctx, schema, req.Table) + if err != nil { + // No catalog: raw strings everywhere, and honour the quoting. + for i := range targets { + conv.textCols[i] = true + } + return conv + } + byName := make(map[string]string, len(cols)) + for _, col := range cols { + byName[strings.ToLower(col.Name)] = col.DataType + } + for i, target := range targets { + dataType, known := byName[strings.ToLower(target)] + conv.textCols[i] = !known || database.AcceptsEmptyString(dataType) + upper := strings.ToUpper(strings.TrimSpace(dataType)) + switch { + case strings.Contains(upper, "BOOL"): + conv.boolCols[i] = true + case reqBool[i] && !conv.textCols[i]: + // 1/0 into numerics; text targets keep the value as written. + conv.boolIntCols[i] = true + } + if mysql { + switch upper { + case "DATE", "DATETIME", "TIMESTAMP": + conv.dateCols[i] = true + } + } + } + return conv +} + // Unrecognised text passes through so the engine reports it rather than this rewriting data. -func normalizeBool(v string) any { +func normalizeBool(v string, asBool bool) any { switch strings.ToLower(strings.TrimSpace(v)) { case "true", "t", "yes", "y", "1": + if asBool { + return true + } return 1 case "false", "f", "no", "n", "0": + if asBool { + return false + } return 0 } return v @@ -520,9 +693,11 @@ func (a *App) runSQLImport(ctx context.Context, em *importEmitter, connectionID } type importEmitter struct { - app *App - importID string - seq int + app *App + importID string + seq int + totalRows int64 + lastEmit time.Time } func (e *importEmitter) nextSeq() int { @@ -531,7 +706,16 @@ func (e *importEmitter) nextSeq() int { return seq } +// Safe to call per row: drops anything arriving inside progressInterval. func (e *importEmitter) progress(processed, inserted, skipped, bytesRead, totalBytes int64) { + if time.Since(e.lastEmit) < progressInterval { + return + } + e.progressNow(processed, inserted, skipped, bytesRead, totalBytes) +} + +func (e *importEmitter) progressNow(processed, inserted, skipped, bytesRead, totalBytes int64) { + e.lastEmit = time.Now() e.app.emit("import:progress", ImportProgressEvent{ Seq: e.nextSeq(), ImportID: e.importID, @@ -540,6 +724,7 @@ func (e *importEmitter) progress(processed, inserted, skipped, bytesRead, totalB Skipped: skipped, BytesRead: bytesRead, TotalBytes: totalBytes, + TotalRows: e.totalRows, }) } diff --git a/internal/app/app_import_test.go b/internal/app/app_import_test.go index d4b28fd..701610f 100644 --- a/internal/app/app_import_test.go +++ b/internal/app/app_import_test.go @@ -282,16 +282,22 @@ func TestResolveMapping(t *testing.T) { func TestNormalizeBool(t *testing.T) { for _, v := range []string{"true", "T", "yes", "Y", "1"} { - if got := normalizeBool(v); got != 1 { - t.Errorf("normalizeBool(%q) = %v, want 1", v, got) + if got := normalizeBool(v, false); got != 1 { + t.Errorf("normalizeBool(%q, false) = %v, want 1", v, got) + } + if got := normalizeBool(v, true); got != true { + t.Errorf("normalizeBool(%q, true) = %v, want true", v, got) } } for _, v := range []string{"false", "F", "no", "N", "0"} { - if got := normalizeBool(v); got != 0 { - t.Errorf("normalizeBool(%q) = %v, want 0", v, got) + if got := normalizeBool(v, false); got != 0 { + t.Errorf("normalizeBool(%q, false) = %v, want 0", v, got) + } + if got := normalizeBool(v, true); got != false { + t.Errorf("normalizeBool(%q, true) = %v, want false", v, got) } } - if got := normalizeBool("maybe"); got != "maybe" { + if got := normalizeBool("maybe", true); got != "maybe" { t.Errorf("normalizeBool(\"maybe\") = %v, want it unchanged", got) } } @@ -322,6 +328,79 @@ func TestPreviewImportFile(t *testing.T) { if preview.Truncated { t.Error("a 2-row file should not report truncation") } + if preview.TotalRows != 2 { + t.Errorf("totalRows = %d, want 2", preview.TotalRows) + } +} + +// The count must cover the whole file, not just the preview sample. +func TestPreviewImportFileCountsEveryRow(t *testing.T) { + var b strings.Builder + b.WriteString("id,name\n") + for i := 1; i <= previewRowLimit*3; i++ { + fmt.Fprintf(&b, "%d,name%d\n", i, i) + } + a, connID, path := importFixture(t, "people.csv", b.String()) + + preview, err := a.PreviewImportFile(connID, path, service.CSVOptions{HasHeader: true}) + if err != nil { + t.Fatalf("PreviewImportFile: %v", err) + } + if len(preview.Rows) != previewRowLimit || !preview.Truncated { + t.Errorf("sample = %d rows, truncated = %v", len(preview.Rows), preview.Truncated) + } + if preview.TotalRows != int64(previewRowLimit*3) { + t.Errorf("totalRows = %d, want %d", preview.TotalRows, previewRowLimit*3) + } +} + +// An unterminated quote is data under lazy quoting, counted like the import will load it. +func TestPreviewImportFileCountsLazyQuotedRows(t *testing.T) { + csv := "id,name\n1,Alice\n2,\"unterminated\n" + a, connID, path := importFixture(t, "people.csv", csv) + + preview, err := a.PreviewImportFile(connID, path, service.CSVOptions{HasHeader: true}) + if err != nil { + t.Fatalf("PreviewImportFile: %v", err) + } + if preview.TotalRows != 2 { + t.Errorf("totalRows = %d, want 2", preview.TotalRows) + } +} + +func TestPreviewImportFileHeaderOnlyCountsNoRows(t *testing.T) { + a, connID, path := importFixture(t, "people.csv", "id,name\n") + preview, err := a.PreviewImportFile(connID, path, service.CSVOptions{HasHeader: true}) + if err != nil { + t.Fatalf("PreviewImportFile: %v", err) + } + if preview.TotalRows != 0 { + t.Errorf("totalRows = %d, want 0", preview.TotalRows) + } +} + +// The row target rides on every progress event. +func TestImportCSVReportsRowTarget(t *testing.T) { + csv := "id,name\n1,Alice\n2,Bob\n3,Carol\n" + a, connID, path := importFixture(t, "people.csv", csv) + mustExecSQL(t, a, connID, "CREATE TABLE people (id INTEGER, name TEXT)") + + targets, sources, err := resolveMapping([]string{"id", "name"}) + if err != nil { + t.Fatalf("resolveMapping: %v", err) + } + em := &importEmitter{app: a, importID: "test"} + if _, err = a.runCSVImport(context.Background(), em, connID, CSVImportRequest{ + Path: path, + Table: "people", + Options: service.CSVOptions{HasHeader: true}, + Mapping: []string{"id", "name"}, + }, targets, sources); err != nil { + t.Fatalf("runCSVImport: %v", err) + } + if em.totalRows != 3 { + t.Errorf("emitter totalRows = %d, want 3", em.totalRows) + } } func TestPreviewImportFileWithoutHeader(t *testing.T) { @@ -449,3 +528,92 @@ func mustExecSQL(t *testing.T, a *App, connID, sql string) { t.Fatalf("exec %q: %v", sql, err) } } + +// Bare empty field → NULL; quoted → ”. This is what re-imports into a NOT NULL column. +func TestImportCSVKeepsQuotedEmptyStringApartFromNull(t *testing.T) { + csv := "id,a,b\n1,,\"\"\n" + a, connID, path := importFixture(t, "blanks.csv", csv) + mustExecSQL(t, a, connID, "CREATE TABLE t (id INTEGER, a TEXT, b TEXT NOT NULL)") + + result := runCSV(t, a, connID, CSVImportRequest{ + Path: path, + Table: "t", + Options: service.CSVOptions{HasHeader: true}, + Mapping: []string{"id", "a", "b"}, + }) + if result.Inserted != 1 || result.Skipped != 0 { + t.Fatalf("result = %+v, want 1 inserted", result) + } + rows := queryAll(t, a, connID, "SELECT a IS NULL, b IS NULL, b FROM t") + if rows[0][0] != "1" { + t.Errorf("bare empty field should be NULL, got %v", rows[0]) + } + if rows[0][1] != "0" || rows[0][2] != "" { + t.Errorf("quoted empty field should be an empty string, got %v", rows[0]) + } +} + +func TestImportCSVStripsFormulaGuardApostrophe(t *testing.T) { + csv := "v\n'=1+1\n'@handle\n'-not a number\n'plain\nO'Brien\n" + a, connID, path := importFixture(t, "formulas.csv", csv) + mustExecSQL(t, a, connID, "CREATE TABLE t (v TEXT)") + + result := runCSV(t, a, connID, CSVImportRequest{ + Path: path, + Table: "t", + Options: service.CSVOptions{HasHeader: true}, + Mapping: []string{"v"}, + }) + if result.Inserted != 5 { + t.Fatalf("result = %+v", result) + } + rows := queryAll(t, a, connID, "SELECT v FROM t") + got := make([]string, 0, len(rows)) + for _, r := range rows { + got = append(got, r[0]) + } + want := []string{"=1+1", "@handle", "-not a number", "'plain", "O'Brien"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("values = %v, want %v", got, want) + } +} + +func TestImportCSVNullLiteralBeatsQuoting(t *testing.T) { + csv := "v\n\"\\N\"\n" + a, connID, path := importFixture(t, "nulls.csv", csv) + mustExecSQL(t, a, connID, "CREATE TABLE t (v TEXT)") + + result := runCSV(t, a, connID, CSVImportRequest{ + Path: path, + Table: "t", + Options: service.CSVOptions{HasHeader: true, NullLiteral: `\N`}, + Mapping: []string{"v"}, + }) + if result.Inserted != 1 { + t.Fatalf("result = %+v", result) + } + rows := queryAll(t, a, connID, "SELECT v IS NULL FROM t") + if rows[0][0] != "1" { + t.Errorf("the NULL literal should win over quoting, got %v", rows[0]) + } +} + +func TestImportCSVBoolShapedTextStaysText(t *testing.T) { + a, connID, path := importFixture(t, "flags.csv", "v\ntrue\nf\n") + mustExecSQL(t, a, connID, "CREATE TABLE t (v TEXT)") + + result := runCSV(t, a, connID, CSVImportRequest{ + Path: path, + Table: "t", + Options: service.CSVOptions{HasHeader: true}, + Mapping: []string{"v"}, + ColumnTypes: []string{"bool"}, + }) + if result.Inserted != 2 { + t.Fatalf("result = %+v", result) + } + rows := queryAll(t, a, connID, "SELECT v FROM t ORDER BY rowid") + if rows[0][0] != "true" || rows[1][0] != "f" { + t.Errorf("values = %v, want [true f]", rows) + } +} diff --git a/internal/app/e2e_roundtrip_test.go b/internal/app/e2e_roundtrip_test.go new file mode 100644 index 0000000..81da682 --- /dev/null +++ b/internal/app/e2e_roundtrip_test.go @@ -0,0 +1,355 @@ +//go:build e2e + +// Export → re-import round trips: NULL vs ” quoting, the formula guard, MySQL datetime literals. +package app + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "xensql/internal/database" + "xensql/internal/service" +) + +// Imports the way the dialog does: self-mapped, with the preview's inferred types. +func importCSVInto(t *testing.T, a *App, connID, schema, table string, columns []string, path string) *ImportResult { + t.Helper() + preview, err := a.PreviewImportFile(connID, path, service.CSVOptions{HasHeader: true}) + if err != nil { + t.Fatalf("preview: %v", err) + } + targets, sources, err := resolveMapping(columns) + if err != nil { + t.Fatalf("resolveMapping: %v", err) + } + em := &importEmitter{app: a, importID: "roundtrip"} + result, err := a.runCSVImport(testCtx(), em, connID, CSVImportRequest{ + Path: path, + Schema: schema, + Table: table, + Options: service.CSVOptions{HasHeader: true}, + Mapping: columns, + ColumnTypes: preview.InferredTypes, + }, targets, sources) + if err != nil { + t.Fatalf("import aborted: %v", err) + } + return result +} + +func writeExportedCSV(t *testing.T, result *database.QueryResult) string { + t.Helper() + csv, err := service.ExportResult(result, "csv") + if err != nil { + t.Fatalf("ExportResult: %v", err) + } + path := filepath.Join(t.TempDir(), "export.csv") + if err := os.WriteFile(path, []byte(csv+"\n"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + return path +} + +func TestRoundTripExportThenImport(t *testing.T) { + for _, e := range allEngines() { + t.Run(e.name, func(t *testing.T) { + a := appForTest(t) + connID := requireEngine(t, a, e) + + src, dst := "e2e_rt_src", "e2e_rt_dst" + boolType := "TINYINT(1)" + if e.driver == database.DriverPostgres { + boolType = "BOOLEAN" + } + ddl := fmt.Sprintf(`CREATE TABLE %%s ( + id INT, + nullable_text TEXT, + empty_text TEXT NOT NULL, + payload %s, + amount DECIMAL(10,2), + flag %s, + note TEXT + )`, e.jsonType, boolType) + for _, tbl := range []string{src, dst} { + mustExecSQL(t, a, connID, "DROP TABLE IF EXISTS "+tbl) + mustExecSQL(t, a, connID, fmt.Sprintf(ddl, tbl)) + } + + mustExecSQL(t, a, connID, fmt.Sprintf( + `INSERT INTO %s (id, nullable_text, empty_text, payload, amount, flag, note) VALUES + (1, NULL, '', '{"a": 1, "b": [true, null]}', 12.34, TRUE, 'plain'), + (2, 'set', 'x', '{"nested": {"k": "v, with comma"}}', NULL, FALSE, ''), + (3, NULL, '', 'null', 0.00, NULL, '-not a number')`, src)) + + cols := []string{"id", "nullable_text", "empty_text", "payload", "amount", "flag", "note"} + selectAll := fmt.Sprintf("SELECT %s FROM %%s ORDER BY id", strings.Join(cols, ", ")) + before, err := a.ExecuteQuery(connID, fmt.Sprintf(selectAll, src)) + if err != nil { + t.Fatalf("select: %v", err) + } + + result := importCSVInto(t, a, connID, e.browseSchema, dst, cols, writeExportedCSV(t, before)) + if result.Skipped > 0 { + t.Fatalf("%d of 3 rows were rejected: %s", result.Skipped, first(result.Errors)) + } + + after, err := a.ExecuteQuery(connID, fmt.Sprintf(selectAll, dst)) + if err != nil { + t.Fatalf("select back: %v", err) + } + if got, want := fmt.Sprintf("%#v", after.Rows), fmt.Sprintf("%#v", before.Rows); got != want { + t.Errorf("round trip changed the rows\n got %s\nwant %s", got, want) + } + }) + } +} + +// One column per case, so a failure names the value shape that broke. +func TestRoundTripValueShapes(t *testing.T) { + cases := []struct { + col string + pgType string + myType string + valExpr string + }{ + {"c_null_text", "TEXT", "TEXT", "NULL"}, + {"c_empty_text", "TEXT", "TEXT", "''"}, + {"c_json_obj", "JSONB", "JSON", `'{"k": "v"}'`}, + {"c_json_arr", "JSONB", "JSON", `'[1, 2, {"a": null}]'`}, + {"c_json_null", "JSONB", "JSON", `'null'`}, + {"c_json_str", "JSONB", "JSON", `'"just a string"'`}, + {"c_json_empty_str", "JSONB", "JSON", `'""'`}, + {"c_json_sqlnull", "JSONB", "JSON", "NULL"}, + {"c_blob", "BYTEA", "BLOB", `'abc'`}, + {"c_bool_on", "BOOLEAN", "TINYINT(1)", "TRUE"}, + {"c_bool_off", "BOOLEAN", "TINYINT(1)", "FALSE"}, + {"c_bool_null", "BOOLEAN", "TINYINT(1)", "NULL"}, + {"c_num_null", "DECIMAL(10,2)", "DECIMAL(10,2)", "NULL"}, + {"c_ts", "TIMESTAMP", "DATETIME", "'2026-08-09 12:34:56'"}, + {"c_ts_null", "TIMESTAMP", "DATETIME", "NULL"}, + {"c_date", "DATE", "DATE", "'2026-08-09'"}, + {"c_dash_text", "TEXT", "TEXT", "'-not a number'"}, + {"c_at_text", "TEXT", "TEXT", "'@handle'"}, + {"c_plus_text", "TEXT", "TEXT", "'+44 20 7946 0000'"}, + {"c_eq_text", "TEXT", "TEXT", "'=1+2'"}, + {"c_ws_text", "TEXT", "TEXT", "' leading spaces'"}, + {"c_quote_text", "TEXT", "TEXT", `'say "hi", twice'`}, + {"c_newline_text", "TEXT", "TEXT", "'two" + "\n" + "lines'"}, + } + + for _, e := range allEngines() { + t.Run(e.name, func(t *testing.T) { + a := appForTest(t) + connID := requireEngine(t, a, e) + + // rid keeps a NULL-only row from exporting as a blank line, which CSV skips. + cols := []string{"rid"} + defs := []string{"rid INT"} + vals := []string{"1"} + for _, c := range cases { + colType := c.myType + if e.driver == database.DriverPostgres { + colType = c.pgType + } + cols = append(cols, c.col) + defs = append(defs, c.col+" "+colType) + vals = append(vals, c.valExpr) + } + + src, dst := "e2e_rt2_src", "e2e_rt2_dst" + for _, tbl := range []string{src, dst} { + mustExecSQL(t, a, connID, "DROP TABLE IF EXISTS "+tbl) + mustExecSQL(t, a, connID, fmt.Sprintf("CREATE TABLE %s (%s)", tbl, strings.Join(defs, ", "))) + } + mustExecSQL(t, a, connID, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + src, strings.Join(cols, ", "), strings.Join(vals, ", "))) + + before, err := a.ExecuteQuery(connID, fmt.Sprintf("SELECT %s FROM %s", strings.Join(cols, ", "), src)) + if err != nil { + t.Fatalf("select: %v", err) + } + + for i, c := range cases { + pair := []string{"rid", c.col} + single := &database.QueryResult{Columns: pair, Rows: [][]any{{before.Rows[0][0], before.Rows[0][i+1]}}} + mustExecSQL(t, a, connID, "DELETE FROM "+dst) + + result := importCSVInto(t, a, connID, e.browseSchema, dst, pair, writeExportedCSV(t, single)) + if result.Skipped > 0 { + t.Errorf("%-18s rejected: %s", c.col, first(result.Errors)) + continue + } + back, bErr := a.ExecuteQuery(connID, fmt.Sprintf("SELECT %s FROM %s", c.col, dst)) + if bErr != nil { + t.Fatalf("read back: %v", bErr) + } + got, want := back.Rows[0][0], before.Rows[0][i+1] + if fmt.Sprintf("%#v", got) != fmt.Sprintf("%#v", want) { + t.Errorf("%-18s changed: %#v -> %#v", c.col, want, got) + } + } + }) + } +} + +// A source that quotes every field must still load: ” only lands where the column can hold one. +func TestImportQuotedEmptyFieldsByColumnType(t *testing.T) { + for _, e := range allEngines() { + t.Run(e.name, func(t *testing.T) { + a := appForTest(t) + connID := requireEngine(t, a, e) + + tsType := "DATETIME" + if e.driver == database.DriverPostgres { + tsType = "TIMESTAMP" + } + tbl := "e2e_quoted_all" + mustExecSQL(t, a, connID, "DROP TABLE IF EXISTS "+tbl) + mustExecSQL(t, a, connID, fmt.Sprintf( + "CREATE TABLE %s (n INT, amount DECIMAL(10,2), when_ts %s, note TEXT)", tbl, tsType)) + + path := filepath.Join(t.TempDir(), "quoted.csv") + if err := os.WriteFile(path, []byte("\"n\",\"amount\",\"when_ts\",\"note\"\n\"1\",\"\",\"\",\"\"\n"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + cols := []string{"n", "amount", "when_ts", "note"} + if result := importCSVInto(t, a, connID, e.browseSchema, tbl, cols, path); result.Skipped > 0 { + t.Fatalf("rejected: %s", first(result.Errors)) + } + + back, err := a.ExecuteQuery(connID, fmt.Sprintf("SELECT amount, when_ts, note FROM %s", tbl)) + if err != nil { + t.Fatalf("read back: %v", err) + } + row := back.Rows[0] + if row[0] != nil || row[1] != nil { + t.Errorf("numeric and date blanks should be NULL, got %#v and %#v", row[0], row[1]) + } + if row[2] != "" { + t.Errorf("a text blank should stay an empty string, got %#v", row[2]) + } + }) + } +} + +// Bool-shaped columns land per target: engine bools, 1/0 into numerics, untouched text. +func TestImportBoolShapedColumns(t *testing.T) { + for _, e := range allEngines() { + t.Run(e.name, func(t *testing.T) { + a := appForTest(t) + connID := requireEngine(t, a, e) + + boolType := "TINYINT(1)" + if e.driver == database.DriverPostgres { + boolType = "BOOLEAN" + } + tbl := "e2e_bool_targets" + mustExecSQL(t, a, connID, "DROP TABLE IF EXISTS "+tbl) + mustExecSQL(t, a, connID, fmt.Sprintf( + "CREATE TABLE %s (rid INT, flag %s, hits INT, note TEXT)", tbl, boolType)) + + csv := "rid,flag,hits,note\n1,1,1,true\n2,0,0,false\n3,true,1,t\n4,false,0,f\n" + path := filepath.Join(t.TempDir(), "bools.csv") + if err := os.WriteFile(path, []byte(csv), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + cols := []string{"rid", "flag", "hits", "note"} + if result := importCSVInto(t, a, connID, e.browseSchema, tbl, cols, path); result.Skipped > 0 { + t.Fatalf("rejected %d rows: %s", result.Skipped, first(result.Errors)) + } + + back, err := a.ExecuteQuery(connID, fmt.Sprintf( + "SELECT COUNT(*) FROM %s WHERE flag = TRUE", tbl)) + if err != nil { + t.Fatalf("read back: %v", err) + } + if got := fmt.Sprint(back.Rows[0][0]); got != "2" { + t.Errorf("true flags = %s, want 2", got) + } + notes, err := a.ExecuteQuery(connID, fmt.Sprintf( + "SELECT note FROM %s ORDER BY rid", tbl)) + if err != nil { + t.Fatalf("read notes: %v", err) + } + want := []string{"true", "false", "t", "f"} + for i, row := range notes.Rows { + if fmt.Sprint(row[0]) != want[i] { + t.Errorf("note[%d] = %#v, want %q (bool-shaped text must stay text)", i, row[0], want[i]) + } + } + }) + } +} + +func first(errs []string) string { + if len(errs) == 0 { + return "(no message)" + } + return errs[0] +} + +// The field-reported failing table, faithful DDL: custom enum arrays, self-FK, PK, unique. +func TestImportPostgresRealWorldRow(t *testing.T) { + e := pgEngine() + a := appForTest(t) + connID := requireEngine(t, a, e) + + tbl := "e2e_users_real" + mustExecSQL(t, a, connID, "DROP TABLE IF EXISTS "+tbl) + mustExecSQL(t, a, connID, "DROP TYPE IF EXISTS e2e_user_scopes CASCADE") + mustExecSQL(t, a, connID, "DROP TYPE IF EXISTS e2e_permissions CASCADE") + mustExecSQL(t, a, connID, "CREATE TYPE e2e_user_scopes AS ENUM ('compliance', 'audit', 'billing')") + mustExecSQL(t, a, connID, "CREATE TYPE e2e_permissions AS ENUM ('read', 'write')") + mustExecSQL(t, a, connID, fmt.Sprintf(`CREATE TABLE %s ( + id integer NOT NULL, + email character varying NOT NULL, + scopes e2e_user_scopes[] NOT NULL, + display_name character varying, + disabled boolean DEFAULT false NOT NULL, + created_at timestamp without time zone, + username text, + updated_at timestamp without time zone, + modified_by integer NOT NULL, + permissions e2e_permissions[] DEFAULT '{}'::e2e_permissions[] NOT NULL, + CONSTRAINT %s_fk FOREIGN KEY (modified_by) REFERENCES %s(id), + CONSTRAINT %s_pk PRIMARY KEY (id), + CONSTRAINT %s_uq UNIQUE (username) + )`, tbl, tbl, tbl, tbl, tbl)) + + csv := "id,email,scopes,display_name,disabled,created_at,username,updated_at,modified_by,permissions\n" + + "8,k.e@example.com,{compliance},Kyle E,true,,kyle.e,2024-04-08T12:07:05.812Z,235,{}\n" + + "235,admin@example.com,\"{audit,billing}\",Admin,false,2023-01-02T03:04:05Z,admin,2024-01-01T00:00:00Z,235,{read}\n" + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte(csv), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + + cols := strings.Split("id,email,scopes,display_name,disabled,created_at,username,updated_at,modified_by,permissions", ",") + if result := importCSVInto(t, a, connID, e.browseSchema, tbl, cols, path); result.Skipped > 0 { + t.Fatalf("rejected %d rows: %s", result.Skipped, first(result.Errors)) + } + + back, err := a.ExecuteQuery(connID, fmt.Sprintf(`SELECT + disabled, created_at IS NULL, scopes[1], cardinality(permissions), + updated_at = '2024-04-08T12:07:05.812Z'::timestamp + FROM %s WHERE id = 8`, tbl)) + if err != nil { + t.Fatalf("read back: %v", err) + } + if got := fmt.Sprint(back.Rows[0]); got != "[true true compliance 0 true]" { + t.Errorf("row 8 = %s, want [true true compliance 0 true]", got) + } + back, err = a.ExecuteQuery(connID, fmt.Sprintf( + "SELECT disabled, scopes[2], permissions[1] FROM %s WHERE id = 235", tbl)) + if err != nil { + t.Fatalf("read back: %v", err) + } + if got := fmt.Sprint(back.Rows[0]); got != "[false billing read]" { + t.Errorf("row 235 = %s, want [false billing read]", got) + } +} diff --git a/internal/app/version.go b/internal/app/version.go index 520ee9c..b4a63b1 100644 --- a/internal/app/version.go +++ b/internal/app/version.go @@ -3,4 +3,4 @@ package app // Version is the source of truth for the application version. // In order to update it use the following command: // go run ./cmd/bump-version [-major, -minor, -patch, 1.2.0] -const Version = "1.5.0" +const Version = "1.5.1" diff --git a/internal/database/importsql.go b/internal/database/importsql.go index 78d0223..57c49f6 100644 --- a/internal/database/importsql.go +++ b/internal/database/importsql.go @@ -25,6 +25,38 @@ var sqlTypeNames = map[ImportColumnType]struct{ postgres, mysql, sqlite string } ImportText: {"text", "TEXT", "TEXT"}, } +// Matched as substrings, so VARCHAR(50), LONGTEXT, NVARCHAR etc. all hit. +var emptyStringTypes = []string{"CHAR", "TEXT", "CLOB", "STRING", "ENUM", "SET", "BINARY", "BLOB", "BYTEA"} + +// AcceptsEmptyString reports whether this type can hold ”; unknown or blank types are assumed to. +func AcceptsEmptyString(dataType string) bool { + upper := strings.ToUpper(strings.TrimSpace(dataType)) + if upper == "" { + return true + } + for _, t := range emptyStringTypes { + if strings.Contains(upper, t) { + return true + } + } + return !knownNonTextType(upper) +} + +// Types that reject ” outright. +var nonTextTypes = []string{ + "INT", "SERIAL", "DECIMAL", "NUMERIC", "FLOAT", "DOUBLE", "REAL", "MONEY", "BIT", + "BOOL", "DATE", "TIME", "YEAR", "JSON", "UUID", "INTERVAL", "XML", "OID", "ARRAY", +} + +func knownNonTextType(upper string) bool { + for _, t := range nonTextTypes { + if strings.Contains(upper, t) { + return true + } + } + return false +} + func SQLTypeFor(driver DriverType, t ImportColumnType) string { names, ok := sqlTypeNames[t] if !ok { diff --git a/internal/database/importsql_test.go b/internal/database/importsql_test.go index fc8e8d3..ae81101 100644 --- a/internal/database/importsql_test.go +++ b/internal/database/importsql_test.go @@ -129,3 +129,40 @@ func TestBuildTruncate(t *testing.T) { t.Errorf("BuildTruncate() = %q", got) } } + +func TestAcceptsEmptyString(t *testing.T) { + cases := map[string]bool{ + "TEXT": true, + "text": true, + "VARCHAR(50)": true, + "character varying": true, + "LONGTEXT": true, + "NVARCHAR(10)": true, + "CHAR(1)": true, + "ENUM('a','b')": true, + "BYTEA": true, + "BLOB": true, + "": true, // SQLite's flexible typing + "widget_status": true, // unknown domain type: assume text-shaped + "INT": false, + "INTEGER": false, + "BIGINT": false, + "SERIAL": false, + "DECIMAL(10,2)": false, + "numeric": false, + "DOUBLE PRECISION": false, + "REAL": false, + "BOOLEAN": false, + "DATE": false, + "DATETIME": false, + "TIMESTAMP": false, + "TIME": false, + "JSONB": false, + "UUID": false, + } + for dataType, want := range cases { + if got := AcceptsEmptyString(dataType); got != want { + t.Errorf("AcceptsEmptyString(%q) = %v, want %v", dataType, got, want) + } + } +} diff --git a/internal/database/statements.go b/internal/database/statements.go index 7a9e5b4..26c72f3 100644 --- a/internal/database/statements.go +++ b/internal/database/statements.go @@ -142,7 +142,7 @@ func blockCommentEnd(s string, i, n int, nested bool) int { return n } -// quoteEnd: index past the closing quote ('' doubling; optional \x escapes), or n when unterminated. +// quoteEnd: index past the closing quote (” doubling; optional \x escapes), or n when unterminated. func quoteEnd(s string, i int, quote byte, backslashEscapes bool, n int) int { i++ for i < n { diff --git a/internal/service/csvimport.go b/internal/service/csvimport.go index 072d0a7..5f722a1 100644 --- a/internal/service/csvimport.go +++ b/internal/service/csvimport.go @@ -2,7 +2,6 @@ package service import ( "bufio" - "encoding/csv" "fmt" "io" "strconv" @@ -15,7 +14,7 @@ type CSVOptions struct { // Delimiter is empty to sniff it. Quoting is always the CSV standard. Delimiter string `json:"delimiter,omitempty"` HasHeader bool `json:"hasHeader"` - // NullLiteral is the text that becomes SQL NULL; blank fields always are. + // NullLiteral is extra text that becomes SQL NULL; a bare empty field already is. NullLiteral string `json:"nullLiteral,omitempty"` // SkipRows drops leading lines before the header is read. SkipRows int `json:"skipRows,omitempty"` @@ -82,7 +81,7 @@ func countFields(line string, delim rune) int { return fields } -func NewCSVReader(r io.Reader, opts CSVOptions) (*csv.Reader, error) { +func NewCSVReader(r io.Reader, opts CSVOptions) (*CSVReader, error) { br := bufio.NewReaderSize(r, 64*1024) if err := stripBOM(br); err != nil { return nil, err @@ -95,13 +94,7 @@ func NewCSVReader(r io.Reader, opts CSVOptions) (*csv.Reader, error) { if err != nil { return nil, err } - reader := csv.NewReader(br) - reader.Comma = delim - // Keep reading a ragged file so the import reports bad rows instead of aborting. - reader.FieldsPerRecord = -1 - reader.LazyQuotes = true - reader.TrimLeadingSpace = opts.TrimSpace - return reader, nil + return &CSVReader{br: br, comma: delim, trim: opts.TrimSpace}, nil } func skipLines(br *bufio.Reader, n int) error { diff --git a/internal/service/csvimport_test.go b/internal/service/csvimport_test.go index 08ec889..14f043c 100644 --- a/internal/service/csvimport_test.go +++ b/internal/service/csvimport_test.go @@ -77,14 +77,14 @@ func TestNewCSVReaderStripsBOMAndSkipsRows(t *testing.T) { if err != nil { t.Fatalf("read header: %v", err) } - if header[0] != "name" { - t.Errorf("header[0] = %q, want %q", header[0], "name") + if header[0].Value != "name" { + t.Errorf("header[0] = %q, want %q", header[0].Value, "name") } row, err := reader.Read() if err != nil { t.Fatalf("read row: %v", err) } - if row[0] != "Alice" || row[1] != "30" { + if row[0].Value != "Alice" || row[1].Value != "30" { t.Errorf("row = %v", row) } if _, err := reader.Read(); err != io.EOF { @@ -97,8 +97,8 @@ func TestNewCSVReaderSniffsDelimiter(t *testing.T) { if err != nil { t.Fatalf("NewCSVReader: %v", err) } - if reader.Comma != ';' { - t.Errorf("Comma = %q, want ';'", reader.Comma) + if reader.Comma() != ';' { + t.Errorf("Comma = %q, want ';'", reader.Comma()) } rec, _ := reader.Read() if len(rec) != 2 { diff --git a/internal/service/csvreader.go b/internal/service/csvreader.go new file mode 100644 index 0000000..a794674 --- /dev/null +++ b/internal/service/csvreader.go @@ -0,0 +1,222 @@ +package service + +import ( + "bufio" + "io" + "strings" + "unicode" +) + +// CSVField is one parsed field; Quoted tells an empty string ("") apart from an absent value. +type CSVField struct { + Value string + Quoted bool +} + +// Like encoding/csv (LazyQuotes, FieldsPerRecord=-1) but reports quoting; pinned by csvreader_diff_test.go. +type CSVReader struct { + br *bufio.Reader + comma rune + trim bool + // Own pushback: bufio's UnreadRune silently no-ops after a Peek. + pushback []rune + fields []CSVField + sb strings.Builder +} + +func (r *CSVReader) Comma() rune { return r.comma } + +func CSVValues(fields []CSVField) []string { + out := make([]string, len(fields)) + for i, f := range fields { + out[i] = f.Value + } + return out +} + +// The returned slice is reused across calls. +func (r *CSVReader) Read() ([]CSVField, error) { + for { + fields, blank, err := r.readRecord() + if err != nil { + return nil, err + } + // encoding/csv drops blank lines. + if blank { + continue + } + return fields, nil + } +} + +func (r *CSVReader) readRecord() (fields []CSVField, blank bool, err error) { + r.fields = r.fields[:0] + consumed := false + // A line of only spaces is a one-empty-field record, not a blank line. + spaceSkipped := false + + for { + r.sb.Reset() + + if r.trim { + if err := r.skipLeadingSpace(&spaceSkipped); err != nil && err != io.EOF { + return nil, false, err + } + consumed = consumed || spaceSkipped + } + + c, err := r.readRune() + switch { + case err == io.EOF: + if !consumed && len(r.fields) == 0 { + return nil, false, io.EOF + } + r.fields = append(r.fields, CSVField{Value: ""}) + return r.fields, false, nil + case err != nil: + return nil, false, err + } + consumed = true + + if c == '"' { + done, err := r.readQuoted() + if err != nil { + return nil, false, err + } + r.fields = append(r.fields, CSVField{Value: r.sb.String(), Quoted: true}) + if done { + return r.fields, false, nil + } + continue + } + + r.unreadRune(c) + done, endedBlank, err := r.readBare(len(r.fields) == 0) + if err != nil { + return nil, false, err + } + r.fields = append(r.fields, CSVField{Value: r.sb.String()}) + if done { + return r.fields, endedBlank && !spaceSkipped, nil + } + } +} + +// Mirrors TrimLeadingSpace, which trims even when the delimiter itself is whitespace. +func (r *CSVReader) skipLeadingSpace(skipped *bool) error { + for { + c, err := r.readRune() + if err != nil { + return err + } + // A lone \r is whitespace; \r\n is the record terminator. + if !unicode.IsSpace(c) || c == '\n' || (c == '\r' && r.peekIsNewline()) { + r.unreadRune(c) + return nil + } + *skipped = true + } +} + +func (r *CSVReader) readQuoted() (recordDone bool, err error) { + for { + c, err := r.readRune() + if err == io.EOF { + // LazyQuotes: an unterminated quote ends the field rather than failing. + return true, nil + } + if err != nil { + return false, err + } + if c != '"' { + // encoding/csv folds a \r\n line ending inside a quoted field to \n. + if c == '\r' && r.peekIsNewline() { + _, _ = r.readRune() + r.sb.WriteRune('\n') + continue + } + r.sb.WriteRune(c) + continue + } + next, nErr := r.readRune() + if nErr == io.EOF { + return true, nil + } + if nErr != nil { + return false, nErr + } + switch { + case next == '"': + r.sb.WriteRune('"') + case next == r.comma: + return false, nil + case next == '\n': + return true, nil + case next == '\r' && r.peekIsNewline(): + _, _ = r.readRune() + return true, nil + default: + // LazyQuotes: a bare quote mid-field is data. + r.sb.WriteRune('"') + r.unreadRune(next) + } + } +} + +// blankLine marks a lone-terminator record. +func (r *CSVReader) readBare(firstField bool) (recordDone, blankLine bool, err error) { + empty := true + for { + c, err := r.readRune() + if err == io.EOF { + return true, false, nil + } + if err != nil { + return false, false, err + } + switch { + case c == r.comma: + return false, false, nil + case c == '\n': + return true, firstField && empty, nil + case c == '\r' && r.peekIsNewline(): + _, _ = r.readRune() + return true, firstField && empty, nil + default: + r.sb.WriteRune(c) + empty = false + } + } +} + +func (r *CSVReader) readRune() (rune, error) { + if n := len(r.pushback); n > 0 { + c := r.pushback[n-1] + r.pushback = r.pushback[:n-1] + return c, nil + } + c, _, err := r.br.ReadRune() + if err == nil && c == '\r' { + // encoding/csv drops a trailing \r before EOF. + if _, peekErr := r.br.Peek(1); peekErr != nil { + if peekErr == io.EOF { + return 0, io.EOF + } + return 0, peekErr + } + } + return c, err +} + +func (r *CSVReader) unreadRune(c rune) { + r.pushback = append(r.pushback, c) +} + +func (r *CSVReader) peekIsNewline() bool { + c, err := r.readRune() + if err != nil { + return false + } + r.unreadRune(c) + return c == '\n' +} diff --git a/internal/service/csvreader_diff_test.go b/internal/service/csvreader_diff_test.go new file mode 100644 index 0000000..6b4905c --- /dev/null +++ b/internal/service/csvreader_diff_test.go @@ -0,0 +1,175 @@ +package service + +import ( + "encoding/csv" + "fmt" + "io" + "math/rand" + "strings" + "testing" +) + +func readAllCSVReader(t *testing.T, in string, opts CSVOptions) [][]string { + t.Helper() + r, err := NewCSVReader(strings.NewReader(in), opts) + if err != nil { + t.Fatalf("NewCSVReader: %v", err) + } + var out [][]string + for { + fields, err := r.Read() + if err == io.EOF { + return out + } + if err != nil { + t.Fatalf("Read: %v", err) + } + rec := make([]string, len(fields)) + for i, f := range fields { + rec[i] = f.Value + } + out = append(out, rec) + } +} + +// The exact encoding/csv configuration the import used before. +func readAllStdlib(in string, comma rune, trim bool) ([][]string, error) { + r := csv.NewReader(strings.NewReader(in)) + r.Comma = comma + r.FieldsPerRecord = -1 + r.LazyQuotes = true + r.TrimLeadingSpace = trim + var out [][]string + for { + rec, err := r.Read() + if err == io.EOF { + return out, nil + } + if err != nil { + return out, err + } + cp := make([]string, len(rec)) + copy(cp, rec) + out = append(out, cp) + } +} + +func sameRecords(a, b [][]string) bool { + return fmt.Sprintf("%q", a) == fmt.Sprintf("%q", b) +} + +var csvDiffCases = []string{ + "", + "\n", + "\n\n\n", + "a,b,c", + "a,b,c\n", + "a,b,c\r\n", + "a,b,c\n\n", + "a,b\nc,d\n", + ",", + ",,\n", + `"",,""` + "\n", + `a,"",b` + "\n", + `"a","b"`, + `"a,b",c`, + `"a""b",c`, + `"multi +line",x`, + "\"crlf\r\ninside\",x\n", + `"unterminated,x`, + `"ab"c,d`, + `a"b,c`, + `"a"b"c",d`, + " a, b\n", + "\ta\t,b\n", + "a,b\r", + "a\rb,c\n", + "trailing,\n", + "trailing,", + " \n", + " , \n", + `\.` + "\n", + `"\."` + "\n", + "a,b\n\nc,d\n", + "ragged,row,here\nshort\n", + "quoted\"in\"middle,x\n", + `"",` + "\n", + "\xef\xbb\xbfa,b\n", +} + +func TestCSVReaderMatchesStdlib(t *testing.T) { + for _, trim := range []bool{false, true} { + for _, comma := range []rune{',', ';', '\t', '|'} { + for _, in := range csvDiffCases { + src := in + if comma != ',' { + src = strings.ReplaceAll(in, ",", string(comma)) + } + opts := CSVOptions{Delimiter: string(comma), TrimSpace: trim} + if comma == '\t' { + opts.Delimiter = "\t" + } + want, err := readAllStdlib(stripBOMString(src), comma, trim) + if err != nil { + // The stdlib config is tolerant; anything it still rejects is out of scope. + continue + } + got := readAllCSVReader(t, src, opts) + if !sameRecords(got, want) { + t.Errorf("comma=%q trim=%v input=%q\n got=%q\nwant=%q", comma, trim, src, got, want) + } + } + } + } +} + +// NewCSVReader strips the BOM; the stdlib comparison must see the same bytes. +func stripBOMString(s string) string { + return strings.TrimPrefix(s, "\xef\xbb\xbf") +} + +func TestCSVReaderMatchesStdlibOnRandomInput(t *testing.T) { + // Fixed seed: reproducible corpus. + rng := rand.New(rand.NewSource(20260809)) + alphabet := []string{"a", "b", ",", `"`, "\n", "\r\n", "\r", " ", "\t", `""`, `\.`, "é"} + + for i := 0; i < 25000; i++ { + var b strings.Builder + for n := rng.Intn(20); n > 0; n-- { + b.WriteString(alphabet[rng.Intn(len(alphabet))]) + } + src := b.String() + trim := i%2 == 0 + + want, err := readAllStdlib(src, ',', trim) + if err != nil { + continue + } + got := readAllCSVReader(t, src, CSVOptions{Delimiter: ",", TrimSpace: trim}) + if !sameRecords(got, want) { + t.Fatalf("trim=%v input=%q\n got=%q\nwant=%q", trim, src, got, want) + } + } +} + +func TestCSVReaderReportsQuoting(t *testing.T) { + r, err := NewCSVReader(strings.NewReader(`a,,"", "x" ,"y"`+"\n"), CSVOptions{Delimiter: ","}) + if err != nil { + t.Fatalf("NewCSVReader: %v", err) + } + fields, err := r.Read() + if err != nil { + t.Fatalf("Read: %v", err) + } + want := []CSVField{ + {Value: "a"}, + {Value: ""}, + {Value: "", Quoted: true}, + {Value: ` "x" `}, + {Value: "y", Quoted: true}, + } + if fmt.Sprintf("%#v", fields) != fmt.Sprintf("%#v", want) { + t.Errorf("fields = %#v, want %#v", fields, want) + } +} diff --git a/internal/service/export.go b/internal/service/export.go index 3fe4d52..e7fd630 100644 --- a/internal/service/export.go +++ b/internal/service/export.go @@ -2,11 +2,11 @@ package service import ( "bytes" - "encoding/csv" "encoding/json" "fmt" "strconv" "strings" + "unicode" "xensql/internal/database" ) @@ -83,27 +83,43 @@ func exportJSON(result *database.QueryResult) (string, error) { return strings.TrimRight(b.String(), "\n"), nil } +// Hand-escaped: NULL bare, ” quoted - mirrors csvFormatter in exportResult.ts. func exportCSV(result *database.QueryResult) (string, error) { var b strings.Builder - w := csv.NewWriter(&b) - if err := w.Write(result.Columns); err != nil { - return "", err - } + writeCSVRecord(&b, result.Columns) for _, row := range result.Rows { + b.WriteByte('\n') record := make([]string, len(row)) for i, v := range row { - record[i] = sanitizeCSVCell(cellString(v)) + if v == nil { + record[i] = "" + continue + } + record[i] = escapeCSVCell(sanitizeCSVCell(fmt.Sprint(v))) } - if err := w.Write(record); err != nil { - return "", err + b.WriteString(strings.Join(record, ",")) + } + return b.String(), nil +} + +func writeCSVRecord(b *strings.Builder, fields []string) { + for i, f := range fields { + if i > 0 { + b.WriteByte(',') } + b.WriteString(escapeCSVCell(f)) } - w.Flush() - if err := w.Error(); err != nil { - return "", err +} + +func escapeCSVCell(s string) string { + if s == "" || s == `\.` || strings.ContainsAny(s, ",\"\n\r") || startsWithSpace(s) { + return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` } - // csv.Writer always appends a trailing newline; drop it to match the TS exporter. - return strings.TrimSuffix(b.String(), "\n"), nil + return s +} + +func startsWithSpace(s string) bool { + return s != "" && unicode.IsSpace(rune(s[0])) } func exportSQL(result *database.QueryResult) string { diff --git a/internal/service/export_test.go b/internal/service/export_test.go index bbee2fe..312da89 100644 --- a/internal/service/export_test.go +++ b/internal/service/export_test.go @@ -275,3 +275,19 @@ func TestExportFormatIsCaseInsensitive(t *testing.T) { t.Fatalf("JSON (uppercase) should work: %v", err) } } + +// Both exporters: NULL bare, ” quoted. +func TestExportCSVDistinguishesNullFromEmptyString(t *testing.T) { + result := &database.QueryResult{ + Columns: []string{"a", "b"}, + Rows: [][]any{{nil, ""}}, + } + out, err := ExportResult(result, "csv") + if err != nil { + t.Fatalf("ExportResult: %v", err) + } + want := "a,b\n,\"\"" + if out != want { + t.Errorf("csv = %q, want %q", out, want) + } +}