From 0928a3958daf9473cd0d797e0f8d0eaf203740a7 Mon Sep 17 00:00:00 2001 From: Brandon Corbett Date: Mon, 5 Oct 2026 13:16:05 -0400 Subject: [PATCH 1/2] feat(migrate): import users from a CSV export Adds seamless migrate csv, which maps CSV columns onto the user import contract, validates rows with @seamless-auth/types, sends them in batches of 200 and writes a report of every row's outcome. Dry run by default. --- .changeset/migrate-csv.md | 5 + README.md | 41 ++++ package-lock.json | 26 ++- package.json | 1 + resources/coverage-badge.svg | 8 +- src/commands/helpTopics.ts | 37 ++++ src/commands/migrate.test.ts | 161 +++++++++++++++ src/commands/migrate.ts | 177 +++++++++++++++++ src/core/csv.test.ts | 46 +++++ src/core/csv.ts | 104 ++++++++++ src/core/migrate.test.ts | 213 ++++++++++++++++++++ src/core/migrate.ts | 374 +++++++++++++++++++++++++++++++++++ src/index.ts | 6 + 13 files changed, 1193 insertions(+), 6 deletions(-) create mode 100644 .changeset/migrate-csv.md create mode 100644 src/commands/migrate.test.ts create mode 100644 src/commands/migrate.ts create mode 100644 src/core/csv.test.ts create mode 100644 src/core/csv.ts create mode 100644 src/core/migrate.test.ts create mode 100644 src/core/migrate.ts diff --git a/.changeset/migrate-csv.md b/.changeset/migrate-csv.md new file mode 100644 index 0000000..2dfecfb --- /dev/null +++ b/.changeset/migrate-csv.md @@ -0,0 +1,5 @@ +--- +"seamless-cli": minor +--- + +Add `seamless migrate csv ` to import users from a CSV export into an instance. It is a dry run unless `--apply` is passed, finds columns by header (or a `--map` file kept beside the export), validates each row locally with the shared `@seamless-auth/types` schema, sends rows in batches of 200, and writes a CSV and JSON report of every row's outcome. Exits 1 when any row is rejected or invalid. Needs an auth server with `POST /admin/users/import`. diff --git a/README.md b/README.md index f1a5f72..ebcdbc8 100644 --- a/README.md +++ b/README.md @@ -563,6 +563,47 @@ the exception: it only exists after the code is sent, so that step genuinely nee --- +## Migrating users from another system + +`seamless migrate csv` imports users from a CSV export (an HR system, a directory export, a +spreadsheet) into the instance you are signed in to with `seamless profile login`. It needs an admin +role and an auth server with `POST /admin/users/import`. + +```bash +seamless migrate csv users.csv # dry run: nothing is written +seamless migrate csv users.csv --apply # import +``` + +It is a dry run unless you pass `--apply`, and it always writes a report next to the input +(`users.migrate-plan.csv` and `.json`, or `users.migrate-result.*` after `--apply`) with one line per +row: created, updated, unchanged, rejected (with the server's reason) or invalid. Exits 1 when any +row is rejected or invalid. + +Columns are found by header: `email` (required), `externalId` (or `id`, `employee id`), `phone`, +`roles`, and `organizations` (or `department`). Cells holding several values are split on `;`. +Organizations are slugs or ids and must already exist. When the headers are different, keep a +mapping file beside the export so the import can be re-run the same way: + +```json +{ + "source": "hr-export", + "columns": { "email": "Work Email", "externalId": "Employee ID", "organizations": "Dept" }, + "separator": "|", + "roles": ["staff"], + "organizationRoles": ["member"] +} +``` + +```bash +seamless migrate csv export.csv --map hr-mapping.json --apply +``` + +Re-running is safe. People are matched on `source` plus their external id, then on email, so a +second run reports them `unchanged`. Imports carry no passwords: each user signs in for the first +time by registering with the imported email, which proves they control it. Roles and memberships are +only ever added, an existing account's email is never changed, and admin roles are refused (grant +admin to individuals afterwards). + ## What is configured for you Seamless CLI handles the parts that are usually difficult to get right: diff --git a/package-lock.json b/package-lock.json index 9c90c51..bd8f6a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,17 @@ { "name": "seamless-cli", - "version": "0.6.0", + "version": "0.17.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "seamless-cli", - "version": "0.6.0", + "version": "0.17.0", "license": "AGPL-3.0-only", "dependencies": { "@clack/prompts": "^1.0.1", "@napi-rs/keyring": "^1.3.0", + "@seamless-auth/types": "^0.22.0", "adm-zip": "^0.5.16", "kleur": "^4.1.5" }, @@ -1805,6 +1806,18 @@ "win32" ] }, + "node_modules/@seamless-auth/types": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/@seamless-auth/types/-/types-0.22.0.tgz", + "integrity": "sha512-26MKBkZyj1hizuM5YiZ1Ho2BKDAv1yphzu/RiucQpKVBdr5XW8j7cQ0pqY/Ky+MgsLiBXtDIwleqh8FBOG/8Lg==", + "license": "AGPL-3.0-only", + "dependencies": { + "zod": "^4.3.6" + }, + "engines": { + "node": ">=24 <25" + } + }, "node_modules/@types/adm-zip": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.7.tgz", @@ -4091,6 +4104,15 @@ "funding": { "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index e7af0ab..aad8aba 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "dependencies": { "@clack/prompts": "^1.0.1", "@napi-rs/keyring": "^1.3.0", + "@seamless-auth/types": "^0.22.0", "adm-zip": "^0.5.16", "kleur": "^4.1.5" }, diff --git a/resources/coverage-badge.svg b/resources/coverage-badge.svg index 270ff0d..c931b02 100644 --- a/resources/coverage-badge.svg +++ b/resources/coverage-badge.svg @@ -1,5 +1,5 @@ - - coverage: 99.4% + + coverage: 99.2% @@ -17,7 +17,7 @@ coverage coverage - 99.4% - 99.4% + 99.2% + 99.2% diff --git a/src/commands/helpTopics.ts b/src/commands/helpTopics.ts index 95650fc..f4cb69b 100644 --- a/src/commands/helpTopics.ts +++ b/src/commands/helpTopics.ts @@ -379,6 +379,43 @@ users prepare-device-replacement [--force] [--keep-sessions] [--keep-passke }, ], }, + { + name: "migrate", + usage: [ + "seamless migrate csv [--map ] [--source ] [--apply] [--report ] [--json]", + ], + sections: [ + { + heading: "migrate csv ", + body: `Import users from a CSV export into the instance (requires an admin role). +Runs as a dry run unless --apply is passed, and writes a report either way. + +Columns are found by header name: email (required), externalId (or id, +employee id), phone, roles, and organizations (or department). Multi-value +cells are split on ";". Organizations are slugs or ids, and must exist. + +--map + • Name the columns and defaults yourself, for example: + { "source": "hr-export", + "columns": { "email": "Work Email", "externalId": "Employee ID" }, + "separator": "|", "roles": ["staff"], "organizationRoles": ["member"] } +--source + • The system the users come from (default csv). Re-runs with the same + source match people on their externalId, so keep it stable +--apply + • Write to the instance. Without it nothing is written +--report + • Where to write .csv and .json (default: next to the input) +--json + • Print the report as JSON + +Imports carry no passwords. Each user signs in for the first time by +registering with their email. Roles and memberships are only added, never +removed, and admin roles are refused. Exits 1 when any row is rejected or +invalid.`, + }, + ], + }, { name: "org", usage: [ diff --git a/src/commands/migrate.test.ts b/src/commands/migrate.test.ts new file mode 100644 index 0000000..649c060 --- /dev/null +++ b/src/commands/migrate.test.ts @@ -0,0 +1,161 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createAuthClient, type AuthClient } from "../core/authClient.js"; +import { runMigrate } from "./migrate.js"; + +vi.mock("../core/authClient.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, createAuthClient: vi.fn() }; +}); + +class ExitError extends Error { + constructor(readonly code: number) { + super(`process.exit(${code})`); + } +} + +const request = vi.fn(); +const fakeClient = { + profile: { name: "default", instanceUrl: "https://auth.example.com" }, + get: vi.fn(), + post: vi.fn(), + request, +} as unknown as AuthClient; + +let dir: string; +let logSpy: ReturnType; + +function writeCsv(lines: string[]) { + const file = path.join(dir, "users.csv"); + fs.writeFileSync(file, lines.join("\n")); + return file; +} + +function respond(results: unknown[], dryRun: boolean) { + request.mockResolvedValueOnce({ + ok: true, + status: 200, + headers: new Headers(), + data: { + source: "csv", + dryRun, + summary: { created: 0, updated: 0, unchanged: 0, rejected: 0 }, + results, + }, + }); +} + +const output = () => logSpy.mock.calls.map((c) => String(c[0])).join("\n"); + +beforeEach(() => { + vi.clearAllMocks(); + dir = fs.mkdtempSync(path.join(os.tmpdir(), "seamless-migrate-cmd-")); + vi.mocked(createAuthClient).mockResolvedValue(fakeClient); + vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new ExitError(code ?? 0); + }) as never); + logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); +}); + +afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe("seamless migrate csv", () => { + it("is a dry run by default and writes a plan report beside the input", async () => { + const file = writeCsv(["email", "ada@example.com"]); + respond([{ index: 0, email: "ada@example.com", status: "created", changes: ["created"] }], true); + + await runMigrate(["csv", file]); + + const body = JSON.parse(request.mock.calls[0][1].body); + expect(body).toEqual({ source: "csv", dryRun: true, users: [{ email: "ada@example.com" }] }); + expect(fs.existsSync(path.join(dir, "users.migrate-plan.csv"))).toBe(true); + expect(fs.existsSync(path.join(dir, "users.migrate-plan.json"))).toBe(true); + expect(output()).toMatch(/Dry run: nothing was written/); + }); + + it("writes with --apply and names the report as a result", async () => { + const file = writeCsv(["email", "ada@example.com"]); + respond([{ index: 0, email: "ada@example.com", status: "created", userId: "u1" }], false); + + await runMigrate(["csv", file, "--apply", "--source", "hr-export"]); + + expect(JSON.parse(request.mock.calls[0][1].body)).toMatchObject({ + source: "hr-export", + dryRun: false, + }); + expect(fs.existsSync(path.join(dir, "users.migrate-result.json"))).toBe(true); + }); + + it("exits 1 when a row is rejected or invalid, after writing the report", async () => { + const file = writeCsv(["email,roles", "ada@example.com,admin", "nope,"]); + respond( + [{ index: 0, email: "ada@example.com", status: "rejected", reason: "admin_role_not_allowed" }], + true, + ); + + await expect(runMigrate(["csv", file, "--report", path.join(dir, "out")])).rejects.toThrow( + "process.exit(1)", + ); + + const report = JSON.parse(fs.readFileSync(path.join(dir, "out.json"), "utf-8")); + expect(report.summary).toMatchObject({ rejected: 1, invalid: 1 }); + expect(output()).toMatch(/row 2.*admin_role_not_allowed/s); + }); + + it("writes what finished when a later batch fails", async () => { + const lines = ["email", ...Array.from({ length: 201 }, (_, i) => `u${i}@example.com`)]; + const file = writeCsv(lines); + respond( + Array.from({ length: 200 }, (_, index) => ({ + index, + email: `u${index}@example.com`, + status: "created", + })), + false, + ); + request.mockResolvedValueOnce({ ok: false, status: 500, data: null, headers: new Headers() }); + + await expect(runMigrate(["csv", file, "--apply"])).rejects.toThrow("process.exit(1)"); + + const report = JSON.parse( + fs.readFileSync(path.join(dir, "users.migrate-result.json"), "utf-8"), + ); + expect(report.rows).toHaveLength(200); + }); + + it("does not contact the instance when every row is invalid", async () => { + const file = writeCsv(["email", "nope"]); + + await expect(runMigrate(["csv", file])).rejects.toThrow("process.exit(1)"); + + expect(createAuthClient).not.toHaveBeenCalled(); + }); + + it("prints JSON with --json", async () => { + const file = writeCsv(["email", "ada@example.com"]); + respond([{ index: 0, email: "ada@example.com", status: "unchanged" }], true); + + await runMigrate(["csv", file, "--json"]); + + const printed = JSON.parse(String(logSpy.mock.calls.at(-1)![0])); + expect(printed.summary).toMatchObject({ unchanged: 1 }); + }); + + it("rejects an unknown source and a missing file argument", async () => { + await expect(runMigrate(["okta"])).rejects.toThrow("process.exit(1)"); + await expect(runMigrate(["csv"])).rejects.toThrow("process.exit(1)"); + }); + + it("explains a CSV without an email column", async () => { + const file = writeCsv(["name", "Ada"]); + + await expect(runMigrate(["csv", file])).rejects.toThrow("process.exit(1)"); + + expect(vi.mocked(console.error).mock.calls[0][0]).toMatch(/No email column/); + }); +}); diff --git a/src/commands/migrate.ts b/src/commands/migrate.ts new file mode 100644 index 0000000..d7cb03e --- /dev/null +++ b/src/commands/migrate.ts @@ -0,0 +1,177 @@ +import fs from "fs"; +import path from "path"; +import kleur from "kleur"; +import { extractFlag } from "../core/args.js"; +import { createAuthClient } from "../core/authClient.js"; +import { parseCsvRecords } from "../core/csv.js"; +import { + buildReport, + ImportBatchError, + loadMapping, + prepareRows, + resolveColumns, + runImport, + summarize, + writeReport, + type MigrationOutcome, + type ReportRow, +} from "../core/migrate.js"; +import { reportAdminError } from "./adminShared.js"; + +const USAGE = + "Usage: seamless migrate csv [--map ] [--source ] [--apply] [--report ] [--json]"; + +// Shown inline; the full list is always in the report files. +const PROBLEMS_SHOWN = 10; + +export async function runMigrate(args: string[]): Promise { + const sub = args[0]; + if (sub !== "csv") { + console.error(kleur.red(`Unknown migrate source: ${sub ?? "(none)"}`)); + console.log(USAGE); + process.exit(1); + } + + let rest = args.slice(1); + const flag = (name: string) => { + const extracted = extractFlag(rest, name); + rest = extracted.rest; + return extracted.value; + }; + const profileFlag = flag("profile"); + const mapPath = flag("map"); + const sourceFlag = flag("source"); + const reportFlag = flag("report"); + const apply = rest.includes("--apply"); + const json = rest.includes("--json"); + const file = rest.find((arg) => !arg.startsWith("--")); + + if (!file) { + console.error(kleur.red("Name the CSV file to import.")); + console.log(USAGE); + process.exit(1); + } + + let prepared: ReturnType; + let source: string; + let columns: ReturnType; + try { + const mapping = loadMapping(mapPath, { source: sourceFlag }); + const { headers, records } = parseCsvRecords(fs.readFileSync(file, "utf-8")); + if (records.length === 0) { + throw new Error(`${file} has no data rows.`); + } + columns = resolveColumns(headers, mapping); + prepared = prepareRows(records, columns, mapping); + source = mapping.source; + } catch (err) { + console.error(kleur.red((err as Error).message)); + process.exit(1); + } + + const dryRun = !apply; + const reportBase = + reportFlag ?? + path.join( + path.dirname(file), + `${path.basename(file, path.extname(file))}.migrate-${dryRun ? "plan" : "result"}`, + ); + + if (!json) { + console.log( + kleur.bold(dryRun ? "Dry run" : "Importing") + + kleur.dim(` ${file} as source "${source}"`), + ); + for (const [field, header] of Object.entries(columns)) { + console.log(kleur.dim(` ${field.padEnd(13)} <- ${header}`)); + } + if (prepared.invalid.length) { + console.log( + kleur.yellow( + ` ${prepared.invalid.length} row(s) failed validation and will not be sent`, + ), + ); + } + } + + let outcomes: MigrationOutcome[] = []; + let failure: unknown; + try { + if (prepared.prepared.length) { + const client = await createAuthClient({ profileFlag }); + outcomes = await runImport( + client, + source, + prepared.prepared, + dryRun, + json + ? undefined + : (done, total) => console.log(kleur.dim(` sent ${done} of ${total}`)), + ); + } + } catch (err) { + if (!(err instanceof ImportBatchError)) reportAdminError(err); + outcomes = err.outcomes; + failure = err; + } + + const report = buildReport(outcomes, prepared.invalid); + const written = writeReport(reportBase, { source, dryRun, input: file }, report); + const summary = summarize(report); + + if (json) { + console.log(JSON.stringify({ source, dryRun, summary, rows: report }, null, 2)); + } else { + printSummary(summary, report, written, dryRun); + } + + if (failure) reportAdminError(failure); + if (summary.rejected || summary.invalid) process.exit(1); +} + +function printSummary( + summary: ReturnType, + report: ReportRow[], + written: { json: string; csv: string }, + dryRun: boolean, +): void { + const verb = dryRun ? "would be " : ""; + console.log(""); + console.log( + [ + kleur.green(`${summary.created} ${verb}created`), + kleur.cyan(`${summary.updated} ${verb}updated`), + kleur.dim(`${summary.unchanged} unchanged`), + (summary.rejected ? kleur.red : kleur.dim)(`${summary.rejected} rejected`), + (summary.invalid ? kleur.red : kleur.dim)(`${summary.invalid} invalid`), + ].join(kleur.dim(" · ")), + ); + + const problems = report.filter((r) => r.status === "rejected" || r.status === "invalid"); + for (const problem of problems.slice(0, PROBLEMS_SHOWN)) { + console.log( + kleur.red(` row ${problem.row}`) + + ` ${problem.email || "(no email)"} ` + + kleur.dim([problem.reason, problem.detail].filter(Boolean).join(": ")), + ); + } + if (problems.length > PROBLEMS_SHOWN) { + console.log(kleur.dim(` ...and ${problems.length - PROBLEMS_SHOWN} more in the report`)); + } + + console.log(""); + console.log(kleur.dim(`Report: ${written.csv}`)); + console.log(kleur.dim(` ${written.json}`)); + + if (dryRun) { + console.log( + kleur.yellow("\nDry run: nothing was written. Re-run with --apply to import."), + ); + } else { + console.log( + kleur.dim( + "\nImported users sign in for the first time by registering with their email.", + ), + ); + } +} diff --git a/src/core/csv.test.ts b/src/core/csv.test.ts new file mode 100644 index 0000000..bff2c1e --- /dev/null +++ b/src/core/csv.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { parseCsv, parseCsvRecords, toCsv } from "./csv.js"; + +describe("parseCsv", () => { + it("handles quoted commas, doubled quotes and embedded newlines", () => { + const rows = parseCsv('a,b,c\n"x, y","say ""hi""","line1\nline2"\n'); + expect(rows).toEqual([ + ["a", "b", "c"], + ["x, y", 'say "hi"', "line1\nline2"], + ]); + }); + + it("accepts CRLF endings, a byte order mark and a missing final newline", () => { + expect(parseCsv("a,b\r\n1,2")).toEqual([ + ["a", "b"], + ["1", "2"], + ]); + }); + + it("refuses a file that ends inside a quote", () => { + expect(() => parseCsv('a\n"unterminated')).toThrow(/quoted field/); + }); +}); + +describe("parseCsvRecords", () => { + it("keys values by trimmed header and numbers rows as a spreadsheet does", () => { + const { headers, records } = parseCsvRecords( + ' email , id\na@example.com, 1\n\n"b@example.com","2\nnote"\n', + ); + expect(headers).toEqual(["email", "id"]); + expect(records).toEqual([ + { row: 2, values: { email: "a@example.com", id: "1" } }, + { row: 4, values: { email: "b@example.com", id: "2\nnote" } }, + ]); + }); + + it("returns nothing for an empty file", () => { + expect(parseCsvRecords("")).toEqual({ headers: [], records: [] }); + }); +}); + +describe("toCsv", () => { + it("quotes only the values that need it", () => { + expect(toCsv([["a", 'b"c', "d,e", undefined, 3]])).toBe('a,"b""c","d,e",,3\n'); + }); +}); diff --git a/src/core/csv.ts b/src/core/csv.ts new file mode 100644 index 0000000..aef5b80 --- /dev/null +++ b/src/core/csv.ts @@ -0,0 +1,104 @@ +// RFC 4180: quoted fields may hold commas, newlines and doubled quotes. Exports from +// HR and directory tools routinely do all three, so a split on commas is not enough. +export function parseCsv(text: string): string[][] { + const input = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let quoted = false; + + for (let i = 0; i < input.length; i++) { + const ch = input[i]; + + if (quoted) { + if (ch === '"') { + if (input[i + 1] === '"') { + field += '"'; + i++; + } else { + quoted = false; + } + } else { + field += ch; + } + continue; + } + + if (ch === '"' && field === "") { + quoted = true; + } else if (ch === ",") { + row.push(field); + field = ""; + } else if (ch === "\n" || ch === "\r") { + if (ch === "\r" && input[i + 1] === "\n") i++; + row.push(field); + rows.push(row); + row = []; + field = ""; + } else { + field += ch; + } + } + + if (quoted) { + throw new Error("The CSV ends inside a quoted field."); + } + + if (field !== "" || row.length > 0) { + row.push(field); + rows.push(row); + } + + return rows; +} + +function isBlank(row: string[]) { + return row.every((value) => value.trim() === ""); +} + +export interface CsvRecord { + /** + * The spreadsheet row, counting the header as row 1. Not the file's line number: + * a quoted field can span lines, and a spreadsheet still shows it as one row. + */ + row: number; + values: Record; +} + +export function parseCsvRecords(text: string): { + headers: string[]; + records: CsvRecord[]; +} { + const [headerRow, ...dataRows] = parseCsv(text); + if (!headerRow || isBlank(headerRow)) { + return { headers: [], records: [] }; + } + + const headers = headerRow.map((h) => h.trim()); + const records: CsvRecord[] = []; + dataRows.forEach((cells, i) => { + if (isBlank(cells)) return; + const values: Record = {}; + headers.forEach((header, col) => { + values[header] = (cells[col] ?? "").trim(); + }); + records.push({ row: i + 2, values }); + }); + + return { headers, records }; +} + +export function toCsv(rows: (string | number | undefined)[][]): string { + return ( + rows + .map((row) => + row + .map((value) => { + const text = value === undefined ? "" : String(value); + return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; + }) + .join(","), + ) + .join("\n") + "\n" + ); +} diff --git a/src/core/migrate.test.ts b/src/core/migrate.test.ts new file mode 100644 index 0000000..b66af0b --- /dev/null +++ b/src/core/migrate.test.ts @@ -0,0 +1,213 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { AuthClient } from "./authClient.js"; +import { parseCsvRecords } from "./csv.js"; +import { + buildReport, + ImportBatchError, + loadMapping, + prepareRows, + resolveColumns, + runImport, + summarize, + writeReport, + type PreparedRow, +} from "./migrate.js"; + +const tmp: string[] = []; +function tmpDir() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "seamless-migrate-")); + tmp.push(dir); + return dir; +} +afterEach(() => { + for (const dir of tmp.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +const defaults = () => loadMapping(undefined); + +describe("loadMapping", () => { + it("defaults the source, separator and lists", () => { + expect(defaults()).toEqual({ + source: "csv", + columns: {}, + separator: ";", + roles: [], + organizationRoles: [], + }); + }); + + it("reads a mapping file and lets --source override it", () => { + const file = path.join(tmpDir(), "map.json"); + fs.writeFileSync( + file, + JSON.stringify({ source: "hr", columns: { email: "Work Email" }, roles: ["staff"] }), + ); + expect(loadMapping(file)).toMatchObject({ + source: "hr", + columns: { email: "Work Email" }, + roles: ["staff"], + }); + expect(loadMapping(file, { source: "entra-id" }).source).toBe("entra-id"); + }); + + it("refuses an unknown column, a bad source and unreadable JSON", () => { + const dir = tmpDir(); + const bad = path.join(dir, "bad.json"); + fs.writeFileSync(bad, JSON.stringify({ columns: { password: "Password" } })); + expect(() => loadMapping(bad)).toThrow(/Unknown mapping column "password"/); + expect(() => loadMapping(undefined, { source: "HR Export" })).toThrow(/Invalid source/); + const broken = path.join(dir, "broken.json"); + fs.writeFileSync(broken, "{"); + expect(() => loadMapping(broken)).toThrow(/Could not read the mapping file/); + }); +}); + +describe("resolveColumns", () => { + it("recognises common header spellings", () => { + expect( + resolveColumns(["Employee ID", "Work Email", "Mobile", "Department"], defaults()), + ).toEqual({ + externalId: "Employee ID", + email: "Work Email", + phone: "Mobile", + organizations: "Department", + }); + }); + + it("prefers the mapping and checks the header exists", () => { + const mapping = { ...defaults(), columns: { email: "Primary" } }; + expect(resolveColumns(["email", "Primary"], mapping).email).toBe("Primary"); + expect(() => + resolveColumns(["email"], { ...defaults(), columns: { email: "Primary" } }), + ).toThrow(/no such column/); + }); + + it("requires an email column", () => { + expect(() => resolveColumns(["name"], defaults())).toThrow(/No email column/); + }); +}); + +describe("prepareRows", () => { + const csv = [ + "id,email,phone,roles,department", + "e1,ada@example.com,+14155552671,clerk;viewer,parks;2c0d53c2-a541-452b-b71b-54c7f15e5877", + "e2,not-an-email,,,", + "e3,grace@example.com,,,", + ].join("\n"); + + it("builds import rows and reports invalid ones with their row", () => { + const { headers, records } = parseCsvRecords(csv); + const mapping = { ...defaults(), roles: ["staff"], organizationRoles: ["member"] }; + const { prepared, invalid } = prepareRows( + records, + resolveColumns(headers, mapping), + mapping, + ); + + expect(prepared[0]).toEqual({ + row: 2, + user: { + externalId: "e1", + email: "ada@example.com", + phone: "+14155552671", + roles: ["staff", "clerk", "viewer"], + organizations: [ + { slug: "parks", roles: ["member"] }, + { organizationId: "2c0d53c2-a541-452b-b71b-54c7f15e5877", roles: ["member"] }, + ], + }, + }); + expect(prepared[1]).toEqual({ + row: 4, + user: { externalId: "e3", email: "grace@example.com", roles: ["staff"] }, + }); + expect(invalid).toEqual([ + { row: 3, email: "not-an-email", externalId: "e2", detail: expect.stringMatching(/^email:/) }, + ]); + }); +}); + +function client(responses: { ok: boolean; status: number; data: unknown }[]) { + const request = vi.fn(); + for (const r of responses) request.mockResolvedValueOnce({ ...r, headers: new Headers() }); + return { + profile: { name: "default", instanceUrl: "https://auth.example.com" }, + get: vi.fn(), + post: vi.fn(), + request, + } as unknown as AuthClient & { request: ReturnType }; +} + +const rowsOf = (n: number): PreparedRow[] => + Array.from({ length: n }, (_, i) => ({ row: i + 2, user: { email: `u${i}@example.com` } })); + +const okBatch = (count: number) => ({ + ok: true, + status: 200, + data: { + source: "csv", + dryRun: true, + summary: { created: count, updated: 0, unchanged: 0, rejected: 0 }, + results: Array.from({ length: count }, (_, index) => ({ + index, + email: `x${index}@example.com`, + status: "created", + })), + }, +}); + +describe("runImport", () => { + it("sends batches of 200 and maps results back to CSV rows", async () => { + const c = client([okBatch(200), okBatch(50)]); + const outcomes = await runImport(c, "csv", rowsOf(250), true); + + expect(c.request).toHaveBeenCalledTimes(2); + const [, init] = c.request.mock.calls[1]; + const body = JSON.parse(init.body); + expect(body).toMatchObject({ source: "csv", dryRun: true }); + expect(body.users).toHaveLength(50); + expect(outcomes).toHaveLength(250); + expect(outcomes[200].row).toBe(202); + }); + + it("explains an instance without the import route", async () => { + const c = client([{ ok: false, status: 404, data: null }]); + await expect(runImport(c, "csv", rowsOf(1), true)).rejects.toThrow(/does not support user import/); + }); + + it("keeps the outcomes of batches that finished before one failed", async () => { + const c = client([okBatch(200), { ok: false, status: 500, data: null }]); + const err = await runImport(c, "csv", rowsOf(250), false).catch((e) => e); + + expect(err).toBeInstanceOf(ImportBatchError); + expect(err.outcomes).toHaveLength(200); + expect(err.message).toMatch(/after 200 of 250 rows/); + }); +}); + +describe("reports", () => { + it("merges results and invalid rows in row order and writes both files", () => { + const report = buildReport( + [ + { row: 4, result: { index: 1, email: "b@example.com", status: "rejected", reason: "phone_in_use" } }, + { row: 2, result: { index: 0, email: "=cmd@example.com", status: "created", changes: ["created"] } }, + ], + [{ row: 3, email: "bad", detail: "email: Invalid email" }], + ); + expect(report.map((r) => r.status)).toEqual(["created", "invalid", "rejected"]); + expect(summarize(report)).toEqual({ created: 1, updated: 0, unchanged: 0, rejected: 1, invalid: 1 }); + + const base = path.join(tmpDir(), "users.migrate-plan"); + const written = writeReport(base, { source: "csv", dryRun: true, input: "users.csv" }, report); + + const csv = fs.readFileSync(written.csv, "utf-8"); + expect(csv.split("\n")[0]).toBe("row,email,externalId,status,userId,changes,reason,detail"); + expect(csv).toContain("2,'=cmd@example.com,,created,,created,,"); + const json = JSON.parse(fs.readFileSync(written.json, "utf-8")); + expect(json).toMatchObject({ source: "csv", dryRun: true, summary: { invalid: 1 } }); + expect(json.rows).toHaveLength(3); + }); +}); diff --git a/src/core/migrate.ts b/src/core/migrate.ts new file mode 100644 index 0000000..07ffd2b --- /dev/null +++ b/src/core/migrate.ts @@ -0,0 +1,374 @@ +import fs from "fs"; +import { + ImportUsersResponseSchema, + USER_IMPORT_MAX_ROWS, + UserImportRowSchema, + UserImportSourceSchema, + type ImportUsersResponse, + type UserImportResult, + type UserImportRow, +} from "@seamless-auth/types"; +import { AdminApiError, PermissionError } from "./admin.js"; +import type { AuthClient } from "./authClient.js"; +import type { CsvRecord } from "./csv.js"; +import { toCsv } from "./csv.js"; + +export type MappedField = + | "externalId" + | "email" + | "phone" + | "roles" + | "organizations"; + +const FIELDS: MappedField[] = [ + "externalId", + "email", + "phone", + "roles", + "organizations", +]; + +// Header spellings recognised without a mapping file, compared case-insensitively +// with spaces, underscores and hyphens removed. +const DEFAULT_HEADERS: Record = { + externalId: ["externalid", "id", "employeeid", "userid"], + email: ["email", "emailaddress", "workemail", "mail"], + phone: ["phone", "phonenumber", "mobile", "mobilephone"], + roles: ["roles", "role"], + organizations: ["organizations", "organization", "department", "departments"], +}; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * How a source's columns become import rows. Kept as a file beside the export so a + * migration can be re-run with the same mapping after the source data is corrected. + */ +export interface MigrationMapping { + source: string; + columns: Partial>; + /** Splits multi-value cells (roles, organizations). Defaults to ";". */ + separator: string; + /** Roles added to every row. */ + roles: string[]; + /** Membership roles for each organization a row names. The server defaults to member. */ + organizationRoles: string[]; +} + +function normalizeHeader(header: string) { + return header.toLowerCase().replace(/[\s_-]/g, ""); +} + +function stringList(value: unknown, field: string): string[] { + if (value === undefined) return []; + if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) { + throw new Error(`Mapping field "${field}" must be a list of strings.`); + } + return value as string[]; +} + +export function loadMapping( + path: string | undefined, + overrides: { source?: string } = {}, +): MigrationMapping { + let raw: Record = {}; + if (path) { + try { + raw = JSON.parse(fs.readFileSync(path, "utf-8")); + } catch (err) { + throw new Error(`Could not read the mapping file ${path}: ${(err as Error).message}`); + } + } + + const columns = (raw.columns ?? {}) as Record; + for (const [key, value] of Object.entries(columns)) { + if (!FIELDS.includes(key as MappedField)) { + throw new Error( + `Unknown mapping column "${key}". Expected one of: ${FIELDS.join(", ")}.`, + ); + } + if (typeof value !== "string") { + throw new Error(`Mapping column "${key}" must name a CSV header.`); + } + } + + const source = overrides.source ?? (raw.source as string | undefined) ?? "csv"; + const parsedSource = UserImportSourceSchema.safeParse(source); + if (!parsedSource.success) { + throw new Error( + `Invalid source "${source}": use lowercase letters, digits and hyphens.`, + ); + } + + return { + source: parsedSource.data, + columns: columns as Partial>, + separator: typeof raw.separator === "string" && raw.separator ? raw.separator : ";", + roles: stringList(raw.roles, "roles"), + organizationRoles: stringList(raw.organizationRoles, "organizationRoles"), + }; +} + +/** + * Which CSV header feeds each field: the mapping's choice where it has one, otherwise + * the first header that matches a recognised spelling. Fails when there is no email + * column or when the mapping names a header the file does not have. + */ +export function resolveColumns( + headers: string[], + mapping: MigrationMapping, +): Partial> { + const resolved: Partial> = {}; + + for (const field of FIELDS) { + const chosen = mapping.columns[field]; + if (chosen) { + if (!headers.includes(chosen)) { + throw new Error( + `The mapping sends "${chosen}" to ${field}, but the CSV has no such column.`, + ); + } + resolved[field] = chosen; + continue; + } + const match = headers.find((h) => + DEFAULT_HEADERS[field].includes(normalizeHeader(h)), + ); + if (match) resolved[field] = match; + } + + if (!resolved.email) { + throw new Error( + `No email column found. Name one in the mapping file ("columns": { "email": "
" }).`, + ); + } + + return resolved; +} + +export interface PreparedRow { + row: number; + user: UserImportRow; +} + +export interface InvalidRow { + row: number; + email: string; + externalId?: string; + detail: string; +} + +function split(value: string | undefined, separator: string): string[] { + if (!value) return []; + return value + .split(separator) + .map((part) => part.trim()) + .filter(Boolean); +} + +export function prepareRows( + records: CsvRecord[], + columns: Partial>, + mapping: MigrationMapping, +): { prepared: PreparedRow[]; invalid: InvalidRow[] } { + const prepared: PreparedRow[] = []; + const invalid: InvalidRow[] = []; + const cell = (record: CsvRecord, field: MappedField) => + columns[field] ? record.values[columns[field]!] : undefined; + + for (const record of records) { + const email = cell(record, "email") ?? ""; + const externalId = cell(record, "externalId") || undefined; + const phone = cell(record, "phone") || undefined; + const roles = Array.from( + new Set([...mapping.roles, ...split(cell(record, "roles"), mapping.separator)]), + ); + const organizations = split(cell(record, "organizations"), mapping.separator).map( + (ref) => ({ + ...(UUID.test(ref) ? { organizationId: ref } : { slug: ref }), + ...(mapping.organizationRoles.length ? { roles: mapping.organizationRoles } : {}), + }), + ); + + const candidate = { + email, + ...(externalId ? { externalId } : {}), + ...(phone ? { phone } : {}), + ...(roles.length ? { roles } : {}), + ...(organizations.length ? { organizations } : {}), + }; + + const parsed = UserImportRowSchema.safeParse(candidate); + if (parsed.success) { + prepared.push({ row: record.row, user: parsed.data }); + } else { + const issue = parsed.error.issues[0]; + invalid.push({ + row: record.row, + email, + ...(externalId ? { externalId } : {}), + detail: `${issue.path.join(".") || "row"}: ${issue.message}`, + }); + } + } + + return { prepared, invalid }; +} + +export interface MigrationOutcome { + row: number; + result: UserImportResult; +} + +/** A batch failed after earlier batches were applied. Carries what did complete. */ +export class ImportBatchError extends AdminApiError { + constructor( + message: string, + readonly outcomes: MigrationOutcome[], + ) { + super(message); + this.name = "ImportBatchError"; + } +} + +/** + * Sends the prepared rows in batches the server accepts. Results come back indexed + * within each batch, so they are mapped back onto the CSV row each one came from. + */ +export async function runImport( + client: AuthClient, + source: string, + rows: PreparedRow[], + dryRun: boolean, + onBatch?: (done: number, total: number) => void, +): Promise { + const outcomes: MigrationOutcome[] = []; + + for (let start = 0; start < rows.length; start += USER_IMPORT_MAX_ROWS) { + const batch = rows.slice(start, start + USER_IMPORT_MAX_ROWS); + const res = await client.request("/admin/users/import", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ source, dryRun, users: batch.map((r) => r.user) }), + }); + + if (res.status === 403) throw new PermissionError(); + if (res.status === 404 && outcomes.length === 0) { + throw new AdminApiError( + "This instance does not support user import. Upgrade the auth server to a release with POST /admin/users/import.", + ); + } + if (!res.ok) { + throw new ImportBatchError( + `The import request failed (${res.status}) after ${outcomes.length} of ${rows.length} rows.`, + outcomes, + ); + } + + const parsed = ImportUsersResponseSchema.safeParse(res.data); + if (!parsed.success) { + throw new ImportBatchError( + "The instance returned an import response this CLI cannot read.", + outcomes, + ); + } + + for (const result of parsed.data.results) { + outcomes.push({ row: batch[result.index].row, result }); + } + onBatch?.(Math.min(start + batch.length, rows.length), rows.length); + } + + return outcomes; +} + +export interface ReportRow { + row: number; + email: string; + externalId?: string; + status: UserImportResult["status"] | "invalid"; + userId?: string; + changes?: string[]; + reason?: string; + detail?: string; +} + +export function buildReport( + outcomes: MigrationOutcome[], + invalid: InvalidRow[], +): ReportRow[] { + const rows: ReportRow[] = [ + ...outcomes.map(({ row, result }) => ({ + row, + email: result.email, + ...(result.externalId ? { externalId: result.externalId } : {}), + status: result.status, + ...(result.userId ? { userId: result.userId } : {}), + ...(result.changes ? { changes: result.changes } : {}), + ...(result.reason ? { reason: result.reason } : {}), + ...(result.detail ? { detail: result.detail } : {}), + })), + ...invalid.map((r) => ({ + row: r.row, + email: r.email, + ...(r.externalId ? { externalId: r.externalId } : {}), + status: "invalid" as const, + detail: r.detail, + })), + ]; + return rows.sort((a, b) => a.row - b.row); +} + +export function summarize(report: ReportRow[]) { + const summary = { created: 0, updated: 0, unchanged: 0, rejected: 0, invalid: 0 }; + for (const row of report) summary[row.status] += 1; + return summary; +} + +// A cell starting with one of these is evaluated as a formula when the report is +// opened in a spreadsheet, and the values come from whatever system exported them. +function neutralizeFormula(value: string | undefined) { + return value && /^[=+\-@\t\r]/.test(value) ? `'${value}` : value; +} + +export function writeReport( + basePath: string, + meta: { source: string; dryRun: boolean; input: string }, + report: ReportRow[], +): { json: string; csv: string } { + const json = `${basePath}.json`; + const csv = `${basePath}.csv`; + + fs.writeFileSync( + json, + JSON.stringify( + { + ...meta, + generatedAt: new Date().toISOString(), + summary: summarize(report), + rows: report, + }, + null, + 2, + ) + "\n", + ); + + fs.writeFileSync( + csv, + toCsv([ + ["row", "email", "externalId", "status", "userId", "changes", "reason", "detail"], + ...report.map((r) => [ + r.row, + neutralizeFormula(r.email), + neutralizeFormula(r.externalId), + r.status, + r.userId, + r.changes?.join(";"), + r.reason, + neutralizeFormula(r.detail), + ]), + ]), + ); + + return { json, csv }; +} diff --git a/src/index.ts b/src/index.ts index dc498e7..7305edc 100755 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,7 @@ import { runConfig } from "./commands/config.js"; import { runUsers } from "./commands/users.js"; import { runOrg } from "./commands/org.js"; import { runApps } from "./commands/apps.js"; +import { runMigrate } from "./commands/migrate.js"; import { runTemplates } from "./commands/templates.js"; import { isCancelled } from "./core/cancel.js"; import kleur from "kleur"; @@ -126,6 +127,11 @@ async function main() { return; } + if (command === "migrate") { + await runMigrate(args.slice(1)); + return; + } + unknownCommand(command); } From 69b93726bea61f55227cbab489b7edbda1763f03 Mon Sep 17 00:00:00 2001 From: Brandon Corbett Date: Mon, 5 Oct 2026 19:05:05 -0400 Subject: [PATCH 2/2] chore(deps): bump @seamless-auth/types to 0.24.0 --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index bd8f6a5..70fc433 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@clack/prompts": "^1.0.1", "@napi-rs/keyring": "^1.3.0", - "@seamless-auth/types": "^0.22.0", + "@seamless-auth/types": "^0.24.0", "adm-zip": "^0.5.16", "kleur": "^4.1.5" }, @@ -1807,9 +1807,9 @@ ] }, "node_modules/@seamless-auth/types": { - "version": "0.22.0", - "resolved": "https://registry.npmjs.org/@seamless-auth/types/-/types-0.22.0.tgz", - "integrity": "sha512-26MKBkZyj1hizuM5YiZ1Ho2BKDAv1yphzu/RiucQpKVBdr5XW8j7cQ0pqY/Ky+MgsLiBXtDIwleqh8FBOG/8Lg==", + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@seamless-auth/types/-/types-0.24.0.tgz", + "integrity": "sha512-/S3kZqHZr2KmsmoCZtmCMVUUKKaVQ6NK8Pz5oiwPJImPZ6c+p1EOKWyPAF+clkAfyYLf7f+8m2XvtgDZV+Sw7A==", "license": "AGPL-3.0-only", "dependencies": { "zod": "^4.3.6" diff --git a/package.json b/package.json index aad8aba..e30de73 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "dependencies": { "@clack/prompts": "^1.0.1", "@napi-rs/keyring": "^1.3.0", - "@seamless-auth/types": "^0.22.0", + "@seamless-auth/types": "^0.24.0", "adm-zip": "^0.5.16", "kleur": "^4.1.5" },