From 5e6ef8111001492a69c2869e0b9a794604b84283 Mon Sep 17 00:00:00 2001 From: JasonYv Date: Fri, 9 Oct 2026 18:49:30 +0800 Subject: [PATCH 1/5] feat: put what the shop holds into a site export A site export is how an operator leaves with what is theirs, and the shop handed Mallok nothing for it: variants, prices, the stock ledger, orders and inquiries were in no backup. exportFiles now returns a JSON file for each, under shop/, with a manifest: a row an object, as stored, so that an amount stays whole minor units and nothing a buyer typed is rewritten. A variant carries its product's slug as well as its translation group, which only this database knows. Carts, rates and the scheduler's state stay behind, and of an inquiry the cart it came from and the mark of who sent it. Tables are read with *, and a test fails when a new p_shop_ table is neither exported nor named among those left behind. A table past the row limit fails the export by name rather than be cut short: Mallok then says there is no backup, where a file missing rows would have passed for one. --- scripts/smoke-shop.mjs | 38 ++- src/plugins/shop/index.ts | 5 + src/plugins/shop/lib/export.ts | 195 +++++++++++++++ test/shop/export.test.ts | 435 +++++++++++++++++++++++++++++++++ 4 files changed, 672 insertions(+), 1 deletion(-) create mode 100644 src/plugins/shop/lib/export.ts create mode 100644 test/shop/export.test.ts diff --git a/scripts/smoke-shop.mjs b/scripts/smoke-shop.mjs index 8c02b91..95cc5d9 100644 --- a/scripts/smoke-shop.mjs +++ b/scripts/smoke-shop.mjs @@ -810,8 +810,44 @@ try { `exporting the inquiries returned ${exported.status}: ${csv.slice(0, 300)}`, ); + // 8. The site's export, which is what `mallok export` writes to disk: + // the shop's files are in it beside the content, and no plugin failed — + // Mallok refuses to call an export a backup otherwise. Both plugins + // export inquiries, each under a name of its own. + const siteExport = await ( + await fetch(`${base}/_mallok/api/export`, { + headers: authorised, + signal: timeout(), + }) + ).json(); + const exportedFile = (path) => + (siteExport.files ?? []).find((file) => file.path === path); + const exportedRows = (path) => JSON.parse(exportedFile(path)?.text ?? 'null'); + expect( + Array.isArray(siteExport.pluginExportFailures) && + siteExport.pluginExportFailures.length === 0, + `the export reports a plugin that failed: ${JSON.stringify(siteExport.pluginExportFailures)}`, + ); + expect( + exportedFile('inquiries.csv') !== undefined && + exportedRows('shop/manifest.json')?.format === 1 && + exportedRows('shop/variants.json')?.length === 9 && + exportedRows('shop/variants.json').every( + (variant) => + typeof variant.product_slug === 'string' && + variant.product_slug !== '', + ) && + exportedRows('shop/prices.json')?.length === 27 && + exportedRows('shop/inquiries.json')?.length === 1 && + exportedRows('shop/inquiries.json')[0].inquiry_no === + inquiry.inquiry_no && + exportedRows('shop/inquiries.json')[0].cart_id === undefined && + exportedRows('shop/inquiry-lines.json')?.length === 1, + `the export does not carry the shop as it is: ${JSON.stringify(exportedRows('shop/manifest.json'))}`, + ); + process.stdout.write( - `smoke:shop: ok (settings, publish, seed, every kind of page, ${followed} navigation links in ${site.locales.length} languages, ${shipped.length} theme files, cart, variants in the admin, a cart sent as an inquiry) on ${base}\n`, + `smoke:shop: ok (settings, publish, seed, every kind of page, ${followed} navigation links in ${site.locales.length} languages, ${shipped.length} theme files, cart, variants in the admin, a cart sent as an inquiry, the shop in the site's export) on ${base}\n`, ); } catch (error) { failure = error; diff --git a/src/plugins/shop/index.ts b/src/plugins/shop/index.ts index 9ca99ca..36dc560 100644 --- a/src/plugins/shop/index.ts +++ b/src/plugins/shop/index.ts @@ -15,6 +15,7 @@ import { definePlugin, type PluginContext, } from 'mallok/worker'; +import { exportShopFiles } from './lib/export.js'; import { deleteInquiries, exportInquiries, @@ -65,6 +66,10 @@ export const shop = definePlugin({ remove: (id, ctx) => removeVariant(id, ctx), }, }, + // What the shop holds travels with a site export: leaving Mallok must not + // mean leaving the catalogue, the orders or the inquiries behind. + exportFiles: (ctx: PluginContext) => + exportShopFiles(ctx.db, { defaultLocale: ctx.site.defaultLocale }), routes: { cart: cartPage, 'cart/update': cartUpdate, diff --git a/src/plugins/shop/lib/export.ts b/src/plugins/shop/lib/export.ts new file mode 100644 index 0000000..dd172a7 --- /dev/null +++ b/src/plugins/shop/lib/export.ts @@ -0,0 +1,195 @@ +/** + * What the shop puts into a site export: everything a seller would have to + * take with them, and nothing that only this deployment has a use for. + * + * Mallok asks every plugin for its files when a site is exported, and + * refuses to call the result a backup if one of them fails. These are the + * shop's, under `shop/`: + * + * manifest.json what is in the other files, and how many rows + * variants.json with the slug of the product each belongs to + * prices.json with each variant's SKU + * stock-adjustments.json the ledger of every change of stock + * orders.json, order-lines.json + * payment-events.json every payment acted on, and what was done + * outbox.json what a change of an order still owes + * inquiries.json, inquiry-lines.json + * + * JSON, a row an object, a column a key, as stored: an amount stays the + * integer of minor units it is, an absent value stays `null`, and nothing + * is rewritten for a spreadsheet's sake. The CSV a seller reads inquiries + * from is the admin's own export, which is for reading; this one is for + * carrying. + * + * Not exported: carts, which are a visitor's and gone in a month; exchange + * rates, which are fetched again; the state of the scheduled work; and, of + * an inquiry, the cart it came from and the mark of who sent it — the one + * was a credential and the other means nothing outside this site. + * + * Orders and inquiries hold what a person told the shop about themselves. + * So does this export, which is why Mallok guards it with the `export` + * scope. + */ + +import type { PluginExportFile } from 'mallok/worker'; + +/** + * The most rows of one table an export carries. + * + * An export is built in one request, in memory, and a table without bound — + * the ledger, the orders — would one day be more than a request can build. + * Past this the export fails, by name, rather than leave rows out: Mallok + * then refuses to call the result a backup, which a file missing its oldest + * rows would otherwise pass for. The figure is a guard, not a measurement; + * what a Worker can carry on a real account has not been measured. + */ +export const EXPORT_ROW_LIMIT = 20_000; + +/** The folder every file of the shop's is under. */ +const FOLDER = 'shop'; + +/** The version of the layout above, for whoever reads the files. */ +const FORMAT = 1; + +interface ExportedTable { + /** The file's name, and the table's in the manifest. */ + readonly name: string; + readonly table: string; + /** Ends in a `LIMIT ?`; the default language is bound first when asked. */ + readonly sql: string; + readonly bindsDefaultLocale?: true; + /** Columns that stay behind. */ + readonly without?: readonly string[]; +} + +/** + * Every column of each table, by `*`: a column a later migration adds is in + * the export without anyone remembering to add it here. Each is in an order + * that does not change between two exports of the same data. + */ +const TABLES: readonly ExportedTable[] = [ + { + name: 'variants', + table: 'p_shop_variant', + // A variant belongs to a product by its translation group, an id this + // database gave it. The product's slug is what an export of the content + // calls it, so that the two can be put together again somewhere else. + sql: `SELECT v.*, ( + SELECT c.slug FROM content AS c + WHERE c.translation_group = v.product_group + ORDER BY c.locale = ? DESC, c.locale LIMIT 1 + ) AS product_slug + FROM p_shop_variant AS v ORDER BY v.sku LIMIT ?`, + bindsDefaultLocale: true, + }, + { + name: 'prices', + table: 'p_shop_price', + sql: `SELECT v.sku AS sku, p.* FROM p_shop_price AS p + JOIN p_shop_variant AS v ON v.id = p.variant_id + ORDER BY v.sku, p.currency LIMIT ?`, + }, + { + name: 'stock-adjustments', + table: 'p_shop_stock_adjustment', + sql: 'SELECT * FROM p_shop_stock_adjustment ORDER BY created_at, id LIMIT ?', + }, + { + name: 'orders', + table: 'p_shop_order', + sql: 'SELECT * FROM p_shop_order ORDER BY created_at, id LIMIT ?', + }, + { + name: 'order-lines', + table: 'p_shop_order_line', + sql: 'SELECT * FROM p_shop_order_line ORDER BY order_id, id LIMIT ?', + }, + { + name: 'payment-events', + table: 'p_shop_stripe_event', + sql: 'SELECT * FROM p_shop_stripe_event ORDER BY processed_at, event_id LIMIT ?', + }, + { + name: 'outbox', + table: 'p_shop_outbox', + sql: 'SELECT * FROM p_shop_outbox ORDER BY seq LIMIT ?', + }, + { + name: 'inquiries', + table: 'p_shop_inquiry', + sql: 'SELECT * FROM p_shop_inquiry ORDER BY created_at, id LIMIT ?', + without: ['cart_id', 'ip_hash'], + }, + { + name: 'inquiry-lines', + table: 'p_shop_inquiry_line', + sql: 'SELECT * FROM p_shop_inquiry_line ORDER BY inquiry_id, position LIMIT ?', + }, +]; + +/** The tables an export carries, for a test to hold against the schema. */ +export const EXPORTED_TABLES: readonly string[] = TABLES.map( + (entry) => entry.table, +); + +/** Rows as a file: one to a line, so that two exports can be compared. */ +function fileOf(rows: readonly Record[]): string { + return rows.length === 0 + ? '[]\n' + : `[\n${rows.map((row) => JSON.stringify(row)).join(',\n')}\n]\n`; +} + +/** + * The shop's files for a site export, read in one round trip. + * + * Throws, naming the table, when one has more rows than an export carries. + */ +export async function exportShopFiles( + db: D1Database, + options: { + readonly defaultLocale: string; + readonly rowLimit?: number; + }, +): Promise { + const limit = options.rowLimit ?? EXPORT_ROW_LIMIT; + // One more than the limit is asked for: it is how "too many" is told from + // "exactly as many as there may be". + const results = await db.batch>( + TABLES.map((entry) => + entry.bindsDefaultLocale === true + ? db.prepare(entry.sql).bind(options.defaultLocale, limit + 1) + : db.prepare(entry.sql).bind(limit + 1), + ), + ); + + const counts: Record = {}; + const files = TABLES.map((entry, index): PluginExportFile => { + const rows = results[index]?.results ?? []; + if (rows.length > limit) { + throw new Error( + `The shop's ${entry.name} have more than ${limit} rows, which is more than one export can carry. Nothing of the shop's was exported, so that a file missing rows is not taken for a backup.`, + ); + } + const kept = + entry.without === undefined + ? rows + : rows.map((row) => + Object.fromEntries( + Object.entries(row).filter( + ([column]) => !entry.without?.includes(column), + ), + ), + ); + const path = `${FOLDER}/${entry.name}.json`; + counts[entry.name] = { file: path, rows: kept.length }; + return { path, text: fileOf(kept) }; + }); + + return [ + { + path: `${FOLDER}/manifest.json`, + text: `${JSON.stringify({ format: FORMAT, tables: counts }, null, 2)}\n`, + }, + ...files, + ]; +} diff --git a/test/shop/export.test.ts b/test/shop/export.test.ts new file mode 100644 index 0000000..3efbc9f --- /dev/null +++ b/test/shop/export.test.ts @@ -0,0 +1,435 @@ +import { beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { + EXPORT_ROW_LIMIT, + EXPORTED_TABLES, + exportShopFiles, +} from '../../src/plugins/shop/lib/export.js'; +import { + api, + clearShopTables, + countD1Calls, + createProduct, + createVariant, + db, + ensureSite, + setPrice, + storeInquiry, + type TestProduct, +} from './helpers.js'; + +let screw: TestProduct; +let washer: TestProduct; + +const AT = '2026-10-09T08:00:00.000Z'; + +/** A file of the export, read back as the rows it holds. */ +function rowsOf( + files: readonly { path: string; text: string }[], + name: string, +): Record[] { + const file = files.find((entry) => entry.path === `shop/${name}.json`); + if (file === undefined) { + throw new Error(`The export has no shop/${name}.json`); + } + return JSON.parse(file.text) as Record[]; +} + +function exported(rowLimit?: number) { + return exportShopFiles(db(), { + defaultLocale: 'en', + ...(rowLimit === undefined ? {} : { rowLimit }), + }); +} + +async function table(sql: string): Promise[]> { + const { results } = await db().prepare(sql).all>(); + return results; +} + +/** One of everything the shop holds. */ +async function fill(): Promise { + await createVariant({ + id: 'cs-16', + productGroup: screw.translationGroup, + sku: 'CS-16', + moq: 50, + stock: 0, + stockPolicy: 'made_to_order', + optionValues: { length: '16 mm' }, + leadTime: [10, 15], + }); + await createVariant({ + id: 'cs-10', + productGroup: screw.translationGroup, + sku: 'CS-10', + moq: 100, + stock: 400, + }); + await createVariant({ + id: 'wa-5', + productGroup: washer.translationGroup, + sku: 'WA-5', + }); + await setPrice({ + variantId: 'cs-10', + currency: 'USD', + amountMinor: 185, + source: 'base', + }); + await setPrice({ + variantId: 'cs-10', + currency: 'EUR', + amountMinor: 199, + source: 'auto', + rateUsed: 0.92, + }); + await setPrice({ + variantId: 'cs-16', + currency: 'USD', + amountMinor: 210, + source: 'base', + }); + await db().batch([ + db() + .prepare( + `INSERT INTO p_shop_stock_adjustment + (id, variant_id, delta, reason, ref_id, created_at) + VALUES ('manual:1', 'cs-10', -100, 'manual', NULL, ?), + ('order_paid:o-1:cs-10', 'cs-10', -100, 'order_paid', 'o-1', ?)`, + ) + .bind('2026-10-08T08:00:00.000Z', AT), + db() + .prepare( + `INSERT INTO p_shop_order + (id, order_no, status, currency, subtotal_minor, total_minor, + stripe_payment_intent_id, email, shipping_address, locale, + created_at, updated_at) + VALUES ('o-1', 'ND-261009-7K3M9QXA', 'paid', 'USD', 18500, 18500, + 'pi_1', 'buyer@order.example', ?, 'en', ?, ?)`, + ) + .bind(JSON.stringify({ recipient: 'Ada', country: 'GB' }), AT, AT), + db().prepare( + `INSERT INTO p_shop_order_line + (id, order_id, variant_id, sku_snapshot, name_snapshot, + unit_price_minor, quantity) + VALUES ('o-1:cs-10', 'o-1', 'cs-10', 'CS-10', 'Cap screw M5', 185, 100)`, + ), + db() + .prepare( + `INSERT INTO p_shop_stripe_event + (event_id, type, order_id, payment_intent_id, outcome, processed_at) + VALUES ('evt_1', 'payment_intent.succeeded', 'o-1', 'pi_1', 'paid', ?)`, + ) + .bind(AT), + db() + .prepare( + `INSERT INTO p_shop_outbox (id, topic, order_id, ref, created_at) + VALUES ('order.paid:o-1', 'order.paid', 'o-1', NULL, ?)`, + ) + .bind(AT), + // What an export leaves behind: a cart, a rate, the scheduler's state. + db() + .prepare( + `INSERT INTO p_shop_cart (id, currency, locale, created_at, updated_at, expires_at) + VALUES (?, 'USD', 'en', ?, ?, '2999-01-01')`, + ) + .bind('a'.repeat(32), AT, AT), + db() + .prepare( + "INSERT INTO p_shop_cart_line (cart_id, variant_id, quantity) VALUES (?, 'cs-10', 100)", + ) + .bind('a'.repeat(32)), + db() + .prepare( + `INSERT INTO p_shop_state (key, value, updated_at) + VALUES ('rates_tried_at', ?, ?)`, + ) + .bind(AT, AT), + ]); + await storeInquiry({ + id: 'inq-1', + at: AT, + form: { + name: 'Ada Lovelace', + email: 'ada@buyer.example', + company: '=Engines; Ltd', + message: 'Two lines,\nand a "quote".', + }, + ipHash: 'f'.repeat(64), + }); +} + +describe('the shop in a site export', () => { + beforeAll(async () => { + await ensureSite(); + screw = await createProduct({ title: 'Cap screw', slug: 'export-screw' }); + await createProduct({ + title: 'Zylinderschraube', + slug: 'export-schraube', + locale: 'de', + translationGroup: screw.translationGroup, + }); + // Out in German only: it has no slug in the default language. + washer = await createProduct({ + title: 'Scheibe', + slug: 'export-scheibe', + locale: 'de', + }); + }); + + beforeEach(clearShopTables); + + it('is part of Mallok’s export, with no plugin reported as failed', async () => { + await fill(); + + const response = await api('GET', '/_mallok/api/export'); + const body = (await response.json()) as { + pluginExportFailures: unknown[]; + files: { path: string; text?: string }[]; + }; + + expect(response.status).toBe(200); + // Mallok refuses to call an export a backup when this is not empty. + expect(body.pluginExportFailures).toEqual([]); + expect( + body.files + .map((file) => file.path) + .filter((path) => path.startsWith('shop/')) + .sort(), + ).toEqual([ + 'shop/inquiries.json', + 'shop/inquiry-lines.json', + 'shop/manifest.json', + 'shop/order-lines.json', + 'shop/orders.json', + 'shop/outbox.json', + 'shop/payment-events.json', + 'shop/prices.json', + 'shop/stock-adjustments.json', + 'shop/variants.json', + ]); + const variants = body.files.find( + (file) => file.path === 'shop/variants.json', + ); + expect(JSON.parse(variants?.text ?? '[]')).toHaveLength(3); + }); + + it('carries every row of every table, column for column, as it is stored', async () => { + await fill(); + const files = await exported(); + + for (const [name, sql] of [ + [ + 'stock-adjustments', + 'SELECT * FROM p_shop_stock_adjustment ORDER BY created_at, id', + ], + ['orders', 'SELECT * FROM p_shop_order ORDER BY created_at, id'], + ['order-lines', 'SELECT * FROM p_shop_order_line ORDER BY order_id, id'], + [ + 'payment-events', + 'SELECT * FROM p_shop_stripe_event ORDER BY processed_at, event_id', + ], + ['outbox', 'SELECT * FROM p_shop_outbox ORDER BY seq'], + [ + 'inquiry-lines', + 'SELECT * FROM p_shop_inquiry_line ORDER BY inquiry_id, position', + ], + ] as const) { + const stored = await table(sql); + expect(stored.length, name).toBeGreaterThan(0); + expect(rowsOf(files, name), name).toEqual(stored); + } + // A variant, and with it what an export of the content calls its product. + expect(rowsOf(files, 'variants')).toEqual( + (await table('SELECT * FROM p_shop_variant ORDER BY sku')).map((row) => ({ + ...row, + product_slug: row.id === 'wa-5' ? 'export-scheibe' : 'export-screw', + })), + ); + // A price, and with it the SKU it is a price of. + expect(rowsOf(files, 'prices')).toEqual([ + expect.objectContaining({ + sku: 'CS-10', + currency: 'EUR', + amount_minor: 199, + source: 'auto', + rate_used: 0.92, + }), + expect.objectContaining({ + sku: 'CS-10', + currency: 'USD', + amount_minor: 185, + source: 'base', + rate_used: null, + }), + expect.objectContaining({ + sku: 'CS-16', + currency: 'USD', + amount_minor: 210, + }), + ]); + }); + + it('names a product by its slug in the default language, or in the one it is out in', async () => { + await fill(); + + const bySku = Object.fromEntries( + rowsOf(await exported(), 'variants').map((row) => [ + row.sku, + row.product_slug, + ]), + ); + + expect(bySku).toEqual({ + 'CS-10': 'export-screw', + 'CS-16': 'export-screw', + 'WA-5': 'export-scheibe', + }); + }); + + it('keeps an amount the whole number it is, and what was typed as it was typed', async () => { + await fill(); + const files = await exported(); + + expect(rowsOf(files, 'orders')[0]).toMatchObject({ + total_minor: 18500, + stripe_session_id: null, + shipping_address: '{"recipient":"Ada","country":"GB"}', + }); + // Nothing is rewritten for a spreadsheet's sake: this is for carrying. + expect(rowsOf(files, 'inquiries')[0]).toMatchObject({ + name: 'Ada Lovelace', + company: '=Engines; Ltd', + message: 'Two lines,\nand a "quote".', + subtotal_minor: null, + }); + }); + + it('leaves behind the cart an inquiry came from and the mark of who sent it', async () => { + await fill(); + const files = await exported(); + const [inquiry] = rowsOf(files, 'inquiries'); + const stored = await table('SELECT cart_id, ip_hash FROM p_shop_inquiry'); + + expect(inquiry).not.toHaveProperty('cart_id'); + expect(inquiry).not.toHaveProperty('ip_hash'); + const everything = files.map((file) => file.text).join('\n'); + expect(everything).not.toContain(String(stored[0]?.cart_id)); + expect(everything).not.toContain('f'.repeat(64)); + }); + + it('leaves behind what only this deployment has a use for', async () => { + await fill(); + const files = await exported(); + const everything = files.map((file) => file.text).join('\n'); + + // A visitor's cart, and the id that is all that protects it. + expect(everything).not.toContain('a'.repeat(32)); + expect(everything).not.toContain('rates_tried_at'); + expect(files.map((file) => file.path).join(' ')).not.toMatch( + /cart|rate|state/, + ); + }); + + it('accounts for every table the shop has: exported, or left behind on purpose', async () => { + const all = ( + await table( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'p_shop_%' ORDER BY name", + ) + ).map((row) => row.name); + + // A table a later migration adds has to be put on one of these lists. + expect(all).toEqual( + [ + ...EXPORTED_TABLES, + 'p_shop_cart', + 'p_shop_cart_line', + 'p_shop_rate', + 'p_shop_state', + ].sort(), + ); + }); + + it('says in a manifest what is in each file', async () => { + await fill(); + const files = await exported(); + + expect(files[0]?.path).toBe('shop/manifest.json'); + expect(JSON.parse(files[0]?.text ?? '{}')).toEqual({ + format: 1, + tables: { + variants: { file: 'shop/variants.json', rows: 3 }, + prices: { file: 'shop/prices.json', rows: 3 }, + 'stock-adjustments': { file: 'shop/stock-adjustments.json', rows: 2 }, + orders: { file: 'shop/orders.json', rows: 1 }, + 'order-lines': { file: 'shop/order-lines.json', rows: 1 }, + 'payment-events': { file: 'shop/payment-events.json', rows: 1 }, + outbox: { file: 'shop/outbox.json', rows: 1 }, + inquiries: { file: 'shop/inquiries.json', rows: 1 }, + 'inquiry-lines': { file: 'shop/inquiry-lines.json', rows: 2 }, + }, + }); + }); + + it('exports an empty shop as files that are empty and valid', async () => { + const files = await exported(); + + expect(files).toHaveLength(10); + for (const file of files.slice(1)) { + expect(file.text, file.path).toBe('[]\n'); + } + expect( + Object.values( + ( + JSON.parse(files[0]?.text ?? '{}') as { + tables: Record; + } + ).tables, + ).map((entry) => entry.rows), + ).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 0]); + }); + + it('is the same text for the same data, a row to a line', async () => { + await fill(); + + const first = await exported(); + const second = await exported(); + + expect(second).toEqual(first); + const variants = first.find((file) => file.path === 'shop/variants.json'); + // An opening line, a line for each of the three, a closing line. + expect(variants?.text.trimEnd().split('\n')).toHaveLength(5); + expect(variants?.text.endsWith(']\n')).toBe(true); + }); + + it('reads everything in one round trip', async () => { + await fill(); + + const calls = await countD1Calls(async () => { + await exported(); + }); + + expect(calls).toBe(1); + }); + + it('fails, naming the table, rather than leave rows out', async () => { + await fill(); + + // Three variants: as many as there may be, and one more than that. + await expect(exported(3)).resolves.toHaveLength(10); + await expect(exported(2)).rejects.toThrow( + /^The shop's variants have more than 2 rows, which is more than one export can carry\./, + ); + // And any other table the same: a fourth price, with three variants. + await setPrice({ + variantId: 'cs-16', + currency: 'EUR', + amountMinor: 199, + source: 'manual', + }); + await expect(exported(3)).rejects.toThrow( + /^The shop's prices have more than 3 rows/, + ); + expect(EXPORT_ROW_LIMIT).toBeGreaterThanOrEqual(10_000); + }); +}); From c1b96fc09d5475d2112d96d15e2b956e2caa7030 Mon Sep 17 00:00:00 2001 From: JasonYv Date: Fri, 9 Oct 2026 18:49:30 +0800 Subject: [PATCH 2/5] docs: record the shop's export, what it carries and what it cannot yet do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The design gains §19. The security notes say that an export holds orders and inquiries as stored, and that a shop can outgrow an export built in one request. --- CLAUDE.md | 2 ++ README.md | 1 + README.zh-CN.md | 1 + SECURITY.md | 9 +++++++ .../2026-09-30-nundar-on-mallok-design.md | 27 ++++++++++++++++++- 5 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index a584727..78aa6f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,6 +65,7 @@ Mallok decides the contracts on both sides. Its documentation is the reference: - **An inquiry's emails are a job** (`lib/inquiry-jobs.ts`), queued with `ctx.enqueueStatement` in that batch and given an id and nothing else. It may run twice: each email is marked on the inquiry once it has been handed to Mallok (`notified_at`, `acknowledged_at`), and only what is not marked is sent. The seller's is in English and answered to the buyer; the buyer's, in their language, is sent only when `inquiry_acknowledge` is on — it is off by default because the form has no challenge in front of it. This is the one place the plugin uses Mallok's jobs; the order code still writes its own outbox, which nothing drains. - **Inquiries are read in the admin** in a `table` panel with their lines as a related table, and marked, exported and deleted by the actions in `lib/inquiry-admin.ts`. An amount is stored twice, as minor units and as the text the buyer saw, because a table panel lists a column as it is stored. `cart_id` and `ip_hash` are in the table and not in the panel. - **An inquiry is the first personal data the shop holds.** It goes in no log and no address, and the plugin's own job carries an id only. Every value goes through `escapeHtml` on its way into an email; a name, a company and a phone number are made one line each (white space collapses; it is not refused, because a refused form comes back empty); an address is held to an allow-list (`EMAIL_PATTERN_SOURCE`), because the admin makes a `mailto:` link of it; a cell of the export that a spreadsheet would run as a formula is made text, and so is every piece a semicolon would make a cell of (`inert`). **What the plugin cannot keep to itself is the email**: `ctx.sendEmail` stores the whole message in Mallok's `job` table, where deleting the inquiry does not reach it. `SECURITY.md` says so; do not claim otherwise. +- **What the shop holds is in a site export** (`exportFiles`, `lib/export.ts`; design §19): variants with the slug of their product, prices, the stock ledger, orders and their lines, payment events, the outbox, inquiries and their lines — a JSON file each under `shop/`, a row an object, as stored, with a manifest. Tables are read with `*`, so a column a migration adds is exported without anyone remembering; `test/shop/export.test.ts` fails when a new `p_shop_` table is neither exported nor on its list of those left behind (carts, rates, the scheduler's state). An inquiry's `cart_id` and `ip_hash` stay behind. Nothing is rewritten for a spreadsheet: that is the admin's CSV, which is for reading; this is for carrying. A table over `EXPORT_ROW_LIMIT` fails the export by name rather than leave rows out, because Mallok then refuses to call it a backup — a file missing rows would pass for one. - **Orders and payment are built but not reachable** (design §14). `lib/orders.ts` places an order and confirms its payment, `lib/order-fulfilment.ts` ships, cancels and refunds, `lib/outbox.ts` records what each change still owes, `lib/stripe-webhook.ts` decides what a Stripe delivery means and which status to answer, `lib/stripe-signature.ts` and `lib/stripe-client.ts` talk to Stripe over `fetch`, `lib/order-email.ts` builds the buyer's emails. No route, page or admin screen calls them, and nothing drains the outbox: those wait for Mallok. Do not add a route that works around that. - **A change to an order is conditional on the status it was read in** (`ORDER_STILL_IN_STATUS` in `lib/order-guard.ts`), and the statement that changes the status comes last in its batch. That is what makes a racing second call write nothing. - **A change to an order writes its outbox row in the same batch** (`orderChangedOutbox`). What follows the change — an email, a purge — is owed from that row. Never send from a function's return value: it is lost whenever the Worker stops after the batch. Rows are read back by `seq`, the order they were committed in, never by `created_at`, which is whatever time the caller passed. @@ -105,6 +106,7 @@ When one is built, take it off this list and prove the behaviour with a test or - A challenge cannot be drawn on a plugin's page: a theme may load no script from another host and `afterRender` does not run there, so the inquiry form has no Turnstile in front of it though Mallok could verify a token. What stands in its place is in `SECURITY.md`; do not add a script to get round it. - `ctx.enqueueStatement` takes no condition, so a job is queued beside a conditional write even when the write stored nothing. A job has to find nothing to do in that case. - A page cannot be rendered from a `POST` without being listed under the `POST`'s address, so a form the server refuses comes back with its fields empty. +- A plugin's export is built whole, in memory, in the one request that exports the site: there is no way to hand Mallok a table a page at a time. So the shop's export has a ceiling of rows a table, and a shop past it cannot be exported at all until Mallok offers one. - Nothing tells a plugin of a deletion it missed, and nothing here sweeps up afterwards: a variant with no product may as well be one that was loaded before its product was published, and a sweep cannot tell the two apart. - Switching a plugin on or off, or changing a setting, can leave cached pages as they were — and the admin's "Clear cached pages" can report success without clearing anything. So the plugins are switched on before the first publish (`scripts/lib/local-shop.mjs`, the README), and `test/shop/helpers.ts` deletes the cached home page by hand after changing settings. Keep both until a Mallok release says the purge can be trusted. diff --git a/README.md b/README.md index 1fb3399..e2024a2 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ Nundar builds on `mallok@0.1.0-rc.11`, the first release with the plugin API the | **Pricing** | USD base price; EUR and GBP derived from ECB rates with a buffer, rounding to a price point and a drift threshold; manual prices never overwritten; the pages of a product are purged from the cache when one of its prices moves | | | **Cart** | A form beside each size on a product page, and a cart page in the site's own design and language: set a quantity, remove a line, choose a currency. No script anywhere in it. MOQ, stock and the most a line may hold (500) are enforced by the server, which says what it refused on the cart page | Checkout | | **Inquiries** | The cart sent as one request for a quote, from a form on the cart page: who to answer, and a message. A size with no price can go in the cart to be asked about. The buyer lands on the inquiry's number; the seller is told by email, answerable straight to the buyer, and finds every inquiry in the admin with its lines — to mark, export as CSV, or delete | A challenge in front of the form (it waits for Mallok), which is why the email confirming to the buyer is off unless switched on | +| **Taking your data with you** | Everything the shop holds is in the site's export (`mallok export`), beside the content: variants, prices, the stock ledger, orders, payment events and inquiries, a JSON file each under `shop/`, as stored | An import of those files; an export of a table too large for one request (both wait for Mallok) | | **Orders and payment** | The logic, tested and not yet reachable: orders with line snapshots, a payment that takes stock exactly once however often Stripe reports it, oversold orders, refunds that return stock, Stripe signature checks, order emails in four languages | The checkout and order pages, the webhook route, and order handling in the admin | The reasoning and the plan are in [`docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md`](docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md). diff --git a/README.zh-CN.md b/README.zh-CN.md index 9a32b69..d9e5607 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -60,6 +60,7 @@ Nundar 基于 `mallok@0.1.0-rc.11`，这是第一个带有商城其余部分所 | **定价** | 美元基准价；欧元和英镑按 ECB 汇率换算，带缓冲、价位取整和漂移阈值；手动价格永不被覆盖；某个商品的价格变动时，清除该商品页面的缓存 | | | **购物车** | 商品页每个尺寸旁有一个表单；购物车页沿用站点自己的设计和语言，可改数量、移除、选币种，全程不需要脚本。起订量、库存和单行数量上限（500）由服务端校验，被拒绝的原因直接显示在购物车页上 | 结账 | | **询盘** | 在购物车页填一张表单（回复给谁、留言），把整个购物车作为一张询价单发出。没有价格的尺寸也可以加入购物车来询价。买家提交后看到询盘编号；卖家收到一封邮件，直接回复即可回到买家，并在后台看到每张询盘及其明细，可以标记、导出 CSV 或删除 | 表单前面的人机验证（等 Mallok 提供），因此给买家的确认邮件默认关闭，需要手动开启 | +| **带走你的数据** | 商城持有的全部数据都在站点导出（`mallok export`）里，和内容放在一起：规格、价格、库存台账、订单、付款事件和询盘，各为 `shop/` 下的一个 JSON 文件，按存储原样导出 | 把这些文件再导入；单次请求装不下的大表的导出（两者都要等 Mallok） | | **订单与付款** | 逻辑已写好并有测试，但还没有入口可以用到：带行快照的订单、无论 Stripe 通知多少次都只扣一次库存的付款、超卖订单、退还库存的退款、Stripe 签名校验、四种语言的订单邮件 | 结账页和订单页、webhook 路由、后台的订单处理 | 原因和计划见 [`docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md`](docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md)。 diff --git a/SECURITY.md b/SECURITY.md index e3638a9..48fb19b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -46,6 +46,7 @@ security document describes each. Nundar implements none of them again. | Stripe keys and the webhook signing secret | Not stored yet | They will be plugin secrets, which Mallok encrypts. No code path reads them today. | | What a buyer says of themselves in an inquiry: name, email address, company, phone, message | D1, on the inquiry | Written when a cart is sent as a request for a quote. It is the first personal data the shop holds. Read in the admin by a signed-in administrator, or a token with `export`; deleted there. Never written to a log or put in an address, and the job the plugin queues carries an id only. **The email is another matter**: once handed to Mallok it sits, whole, in Mallok's own queue — see the residual risks. | | A mark of the visitor who sent an inquiry | D1, on the inquiry | Mallok's one-way hash of the connecting address, which is what the limit per visitor counts by. The address itself is never stored, and the admin's list does not show the mark. | +| The shop's part of a site export | Not stored: built when asked | Mallok's export, which needs a signed-in administrator or a token with `export`, includes every order and inquiry — names, email addresses, shipping addresses, messages — as files under `shop/`. Whoever holds an export holds that. It leaves out a cart's id and the mark of an inquiry's sender. | | Cart contents | D1, keyed by an unguessable 128-bit id | Variant ids and quantities only — **never prices**. A test asserts the table has no price column. | | The cart cookie | `HttpOnly`, `SameSite=Lax`, `Secure` on HTTPS, scoped to `/_mallok/p/shop` | It identifies a cart and nothing else. It is not a session and grants nothing in the admin. | | Prices, stock, variants | D1 | Changed only through the admin's variant form — which needs a signed-in administrator, or a token with `content:write` — or the plugin's own scheduled repricing. Every change of stock is a row in a ledger. | @@ -291,6 +292,14 @@ Stated plainly rather than left for an auditor to find: how they can. Whether a browser's autofill ever fills the field no person sees — which would drop a real request without a word — has not been tried. +- **An export is as private as the database.** It carries orders and + inquiries as they are stored. Keep an exported directory where the + database's own backups would be kept, and do not commit one. +- **A shop can outgrow its export.** The export is built in one request; a + table with more rows than `EXPORT_ROW_LIMIT` fails it rather than be cut + short, and Mallok then produces no backup of the site at all. The limit is + a guard that has not been measured on a real account, and nothing prunes + the stock ledger or the orders. It is written up for Mallok. - **Sample content and sample variants are public test data.** Do not load `seed/shop-sample.sql` into a production database. The sample catalogue describes a supplier that does not exist: its certifications, test figures diff --git a/docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md b/docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md index 4ce433c..762b90b 100644 --- a/docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md +++ b/docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md @@ -689,10 +689,35 @@ Not changed: the export still takes up to a thousand inquiries, as Mallok's own 1. *No challenge in front of the form.* Mallok verifies a Turnstile token for a route and has nowhere to draw the widget on a plugin's page. Written up for Mallok. Until then the limits above are what there is, and `SECURITY.md` says what they do not stop. 2. *No retention period, and a copy the shop cannot delete.* An inquiry stays until an administrator deletes it, and its emails stay in Mallok's queue after that (the review's second finding). Written up. -3. *The shop's data is not in Mallok's export.* The plugin implements no `exportFiles`, for inquiries or for anything else it holds — variants, prices, the stock ledger. That is a gap of its own, older than this work, and is not closed here. +3. *The shop's data is not in Mallok's export.* Closed the same day: §19. 4. *A job is queued beside a conditional write that stored nothing* in a true race, and finds nothing to send. Mallok's statement takes no condition. Written up. 5. *An amount is stored twice* because a table panel has no money column. Written up. 6. *Whether a quantity must be a multiple of the minimum order* (§17, left open, item 8) and *the rounding of a derived price* (item 3) were put to the owner with this work and are not settled in it. 7. *The older tests of a race* — a payment reported twice at once, two refunds, a payment against a cancellation, nine in all — were looked at the same day, after the review. They call functions side by side rather than send requests, and such calls do overlap every time: twenty-five rounds of two payments went to the database in one and the same order, both readings before either write. So none had to be rewritten for that. They now run through `atTheSameMoment`, which fails unless the calls overlapped, so that it is seen on every run rather than taken on trust; and each of the nine, with everything that stops its second write taken away, was red eight times in eight. One thing that showed: an order's write is stopped by several keys at once — the status condition, the event's two keys, the ledger row's id, the outbox row's id — and removing the status condition alone turns no payment test red. That is the design, not a gap in the tests. 8. *A request with no connecting address.* Whether a zone that uses Cloudflare's "Remove visitor IP headers" transform still hands a Worker that header is not stated in the documentation that was read. If it does not, every visitor of such a site shares one limit of five an hour. +## 19. What the export changed or found (2026-10-09) + +A site export is how an operator leaves with what is theirs: `mallok export` writes the content, the settings and whatever each plugin hands over, and refuses to call the result a backup when a plugin fails. The shop handed over nothing — a gap as old as the plugin, found while the inquiry cart was built (§18). It hands over everything now. + +**What is exported**, a JSON file each under `shop/`, with a manifest that names each file and counts its rows: variants, prices, the stock ledger, orders and their lines, payment events, the outbox, inquiries and their lines. + +**Choices made here, which the owner may reverse.** + +1. *JSON, as stored.* A row is an object, a column a key; an amount stays the integer of minor units it is, an absent value `null`, and what a buyer typed is not touched. The export of inquiries in the admin is CSV and makes a formula text, because it is for reading in a spreadsheet; this one is for carrying, and a file that had been rewritten for a spreadsheet's sake could not be put back. +2. *A variant carries the slug of its product* as well as the translation group. The group is an id this database gave the product; the slug — in the default language, or in whichever the product is out in — is what the export of the content calls it. +3. *The outbox and the payment events go too.* A row of the outbox that nobody has acted on is a duty — a refund that is owed — and losing it in a move is the worst thing an export could lose. +4. *Left behind*: carts, which are a visitor's and gone in a month; exchange rates, which are fetched again; the scheduler's state; and of an inquiry the cart it came from and the mark of who sent it. +5. *Every column, by `*`.* A column a later migration adds is exported without anyone remembering to list it. A test fails when a new `p_shop_` table is neither exported nor named among those left behind. +6. *A table past twenty thousand rows fails the export, by name.* An export is built whole, in one request. Leaving the oldest rows out would produce a file that passes for a backup; failing makes Mallok say that there is none. The figure is a guard and not a measurement. + +**How it was verified.** Twelve tests inside workerd: the files through Mallok's own export endpoint with no plugin reported as failed; every table's rows compared with the table, column for column; the slug in the default language and outside it; what is left behind, searched for in the whole text; an empty shop; the same text for the same data; one round trip; the limit at its edge for the first table and for another. Thirteen deliberate breakages, twelve red at once and one that called for the second of those. The smoke run reads the shop's nine variants, twenty-seven prices and one inquiry out of the export of a real local Worker, beside the inquiry plugin's own file, and was seen to fail with the export unwired. + +**Not verified.** `mallok export` itself was not run against this site: the test and the smoke run read the endpoint it reads. Nothing was deployed, and what an export costs a Worker in CPU and memory is not known. + +**Left open.** + +1. *Nothing imports these files.* Mallok's import knows content; a plugin has no hook to be handed its own files back. An export without an import is half of taking one's data with one. Written up. +2. *A shop can outgrow its export* (choice 6), and takes the whole site's backup with it when it does. Mallok has no way for a plugin to hand over a table a page at a time. Written up. +3. *Mallok's own inquiry plugin exports its newest thousand rows and no more*, without saying so — found by reading it, while deciding what this export should do past its limit. Written up. + From f242d944219cd468bd6c634b389932cf3f191fe4 Mon Sep 17 00:00:00 2001 From: JasonYv Date: Fri, 9 Oct 2026 19:28:46 +0800 Subject: [PATCH 3/5] test: have the overlap check say what it checks, and refuse what is no race A review showed that atTheSameMoment checked the first trip of every call before the second of any, which is 'every reading before any write' only for a function that reads once. It is told the number of readings now. Two calls that never write, one after the other, look the same as two side by side: that is refused as nothing to race for. And a call that throws before its first wait no longer leaves the others unwaited for. --- test/shop/helpers.test.ts | 54 +++++++++++++++++++++++++++++++++++++++ test/shop/helpers.ts | 45 +++++++++++++++++++++----------- 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/test/shop/helpers.test.ts b/test/shop/helpers.test.ts index dc506b6..66f055e 100644 --- a/test/shop/helpers.test.ts +++ b/test/shop/helpers.test.ts @@ -41,6 +41,60 @@ describe('calls made at the same moment', () => { ).rejects.toThrow(/did not overlap/); }); + it('are refused when none of them wrote: there was nothing to race for', async () => { + // Two calls of one statement each, one after the other, look the same + // as two side by side. Neither is a race. + const once = async (database: D1Database): Promise => { + await database.prepare('SELECT 1').first(); + }; + + await expect(atTheSameMoment([once, once])).rejects.toThrow( + /no write to race for.*0\.0 1\.0/, + ); + }); + + it('take a function that reads twice before it writes, when told so', async () => { + // Both readings sent together: the second trip of each is a reading + // still, and comes after the other's first. + const readsTwice = async (database: D1Database): Promise => { + await Promise.all([ + database.prepare('SELECT 1').first(), + database.prepare('SELECT 2').first(), + ]); + await database.batch([database.prepare('SELECT 3')]); + return 'done'; + }; + + await expect(atTheSameMoment([readsTwice, readsTwice])).rejects.toThrow( + /did not overlap/, + ); + expect( + fulfilledValues( + await atTheSameMoment([readsTwice, readsTwice], { readings: 2 }), + ), + ).toEqual(['done', 'done']); + }); + + it('wait for the others when one throws before it has begun', async () => { + let finished = false; + const settled = await atTheSameMoment([ + async (database) => { + const answer = await readThenWrite(database); + finished = true; + return answer; + }, + (() => { + throw new Error('at once'); + }) as unknown as (database: D1Database) => Promise, + readThenWrite, + ]).catch((error: unknown) => error); + + // The one that never went to the database is said to be late; what + // matters here is that the first had finished by the time that was said. + expect(String(settled)).toMatch(/did not overlap/); + expect(finished).toBe(true); + }); + it('hand back a call that failed as failed, and say so when none may', async () => { const settled = await atTheSameMoment([ readThenWrite, diff --git a/test/shop/helpers.ts b/test/shop/helpers.ts index c145faa..e067a2f 100644 --- a/test/shop/helpers.ts +++ b/test/shop/helpers.ts @@ -602,8 +602,17 @@ function notingDatabase(call: number, order: NotedCall[]): D1Database { /** * Runs calls at the same moment, and fails unless they really overlapped: - * every call had gone to the database for the first time — its reading — - * before any of them went a second time — its write. + * every call had made its readings before any of them made its write. + * + * What it sees is the order in which the calls went to the database, each + * through the database it was handed. A call is taken to read `readings` + * times — once, unless said otherwise — and then to write; so the check is + * that the first `readings` trips of every call came before the trip + * numbered `readings` of any. It is only as good as that description of the + * calls: say how many readings a function makes, and have it use the + * database it is given and no other. A reading made through another handle + * is not seen, and a race between calls that never write is not a race — + * that is refused as well. * * Functions called side by side in one test do overlap that way, every * time: each runs as far as its first wait, which is its reading, before @@ -613,29 +622,35 @@ function notingDatabase(call: number, order: NotedCall[]): D1Database { * test would never be reached, and the test would go on passing. This says * so instead. * - * Each call is given a database of its own to use. For requests through - * `SELF.fetch`, which do not overlap of themselves, see {@link holdWrites}. + * For requests through `SELF.fetch`, which do not overlap of themselves, + * see {@link holdWrites}. */ export async function atTheSameMoment( calls: readonly ((database: D1Database) => Promise)[], + options: { readonly readings?: number } = {}, ): Promise[]> { + const readings = options.readings ?? 1; const order: NotedCall[] = []; const settled = await Promise.allSettled( - calls.map((call, index) => call(notingDatabase(index, order))), + // An async wrapper, so that a call which throws before its first wait + // is one that failed, and the others are still waited for. + calls.map(async (call, index) => call(notingDatabase(index, order))), ); - const firstSecond = order.findIndex((noted) => noted.nth === 1); - const late = calls - .map((_, index) => - order.findIndex((noted) => noted.call === index && noted.nth === 0), - ) - .some( - (first) => first === -1 || (firstSecond !== -1 && first > firstSecond), + const went = order.map((noted) => `${noted.call}.${noted.nth}`).join(' '); + const firstWrite = order.findIndex((noted) => noted.nth === readings); + if (firstWrite === -1) { + throw new Error( + `No call went to the database after its ${readings === 1 ? 'reading' : `${readings} readings`}, so there was no write to race for. They went in this order (call.nth): ${went}`, ); + } + const late = calls.some((_, index) => + Array.from({ length: readings }, (_unused, nth) => + order.findIndex((noted) => noted.call === index && noted.nth === nth), + ).some((at) => at === -1 || at > firstWrite), + ); if (late) { throw new Error( - `The calls did not overlap. They went to the database in this order (call.nth): ${order - .map((noted) => `${noted.call}.${noted.nth}`) - .join(' ')}`, + `The calls did not overlap. They went to the database in this order (call.nth): ${went}`, ); } return settled; From 3c1e1c1f3687a7c00f659da22d03e9ef3f893a30 Mon Sep 17 00:00:00 2001 From: JasonYv Date: Fri, 9 Oct 2026 19:28:46 +0800 Subject: [PATCH 4/5] fix: export every price, with limits a Worker can hold, and other defects a review found - A price whose variant is gone was left out, and the manifest's count with it: the prices were read through a join. Every price is read now, and the column the export adds is named variant_sku, which no table's column is. - Twenty thousand rows a table came, by the review's measure, to more than a Worker's memory, so the limit would never have named a table. It is five thousand rows now, and sixteen megabytes of text for the files together. - Mallok's command line and admin show that the shop failed and not why, so the reason is logged as well. - The shop's settings were in no export. They are in shop/settings.json. - A line separator inside a value broke 'a row to a line' for a reader that splits lines the way Unicode does. It is written as an escape. And four wrong exports passed every test: one dropping columns of an inquiry, one dropping columns of a price, one with no order to its rows, one holding two tables to the limit. The tests compare every table whole, from rows put in the other way round, hold each of the nine to the limit, and list every column of every exported table, so that a new one is looked at before it leaves the shop. --- src/plugins/shop/index.ts | 5 +- src/plugins/shop/lib/export.ts | 96 +++++-- test/shop/export.test.ts | 506 ++++++++++++++++++++++++--------- 3 files changed, 453 insertions(+), 154 deletions(-) diff --git a/src/plugins/shop/index.ts b/src/plugins/shop/index.ts index 36dc560..9cbd61c 100644 --- a/src/plugins/shop/index.ts +++ b/src/plugins/shop/index.ts @@ -69,7 +69,10 @@ export const shop = definePlugin({ // What the shop holds travels with a site export: leaving Mallok must not // mean leaving the catalogue, the orders or the inquiries behind. exportFiles: (ctx: PluginContext) => - exportShopFiles(ctx.db, { defaultLocale: ctx.site.defaultLocale }), + exportShopFiles(ctx.db, { + defaultLocale: ctx.site.defaultLocale, + settings: ctx.settings, + }), routes: { cart: cartPage, 'cart/update': cartUpdate, diff --git a/src/plugins/shop/lib/export.ts b/src/plugins/shop/lib/export.ts index dd172a7..b90a81a 100644 --- a/src/plugins/shop/lib/export.ts +++ b/src/plugins/shop/lib/export.ts @@ -7,6 +7,7 @@ * shop's, under `shop/`: * * manifest.json what is in the other files, and how many rows + * settings.json the shop's own settings, as the admin has them * variants.json with the slug of the product each belongs to * prices.json with each variant's SKU * stock-adjustments.json the ledger of every change of stock @@ -24,7 +25,8 @@ * Not exported: carts, which are a visitor's and gone in a month; exchange * rates, which are fetched again; the state of the scheduled work; and, of * an inquiry, the cart it came from and the mark of who sent it — the one - * was a credential and the other means nothing outside this site. + * was a credential and the other means nothing outside this site. The shop + * stores no secret; were it to, a secret would not be exported either. * * Orders and inquiries hold what a person told the shop about themselves. * So does this export, which is why Mallok guards it with the `export` @@ -34,16 +36,26 @@ import type { PluginExportFile } from 'mallok/worker'; /** - * The most rows of one table an export carries. + * The most rows of one table an export carries, and the most text the + * shop's files may come to together. * - * An export is built in one request, in memory, and a table without bound — - * the ledger, the orders — would one day be more than a request can build. - * Past this the export fails, by name, rather than leave rows out: Mallok - * then refuses to call the result a backup, which a file missing its oldest - * rows would otherwise pass for. The figure is a guard, not a measurement; - * what a Worker can carry on a real account has not been measured. + * An export is built in one request, in memory: the rows as D1 hands them + * over, the files made of them, and Mallok's own copy when it answers. A + * table without bound — the ledger, the orders — would one day be more than + * a Worker's memory. Past either figure the export fails, saying which + * table, rather than leave rows out: Mallok then refuses to call the result + * a backup, which a file missing its oldest rows would otherwise pass for. + * + * Both are guards, not measurements. Five thousand rows of the widest table + * there is — an inquiry with a message as long as one may be — are about + * thirty megabytes as read, which a Worker can hold; twenty thousand, the + * first figure here, were a hundred and ten, which it cannot. What a + * Worker's CPU allowance lets through has not been measured on a real + * account, and on the smallest plan it may well be less than this. */ -export const EXPORT_ROW_LIMIT = 20_000; +export const EXPORT_ROW_LIMIT = 5000; + +export const EXPORT_BYTE_LIMIT = 16 * 1024 * 1024; /** The folder every file of the shop's is under. */ const FOLDER = 'shop'; @@ -85,9 +97,11 @@ const TABLES: readonly ExportedTable[] = [ { name: 'prices', table: 'p_shop_price', - sql: `SELECT v.sku AS sku, p.* FROM p_shop_price AS p - JOIN p_shop_variant AS v ON v.id = p.variant_id - ORDER BY v.sku, p.currency LIMIT ?`, + // Every price, the one of a variant that is gone as well: a row left + // out here would be a row the manifest could not show to be missing. + sql: `SELECT p.*, v.sku AS variant_sku FROM p_shop_price AS p + LEFT JOIN p_shop_variant AS v ON v.id = p.variant_id + ORDER BY p.variant_id, p.currency LIMIT ?`, }, { name: 'stock-adjustments', @@ -132,26 +146,59 @@ export const EXPORTED_TABLES: readonly string[] = TABLES.map( (entry) => entry.table, ); +/** + * A row as one line of JSON, for every reader. `JSON.stringify` leaves the + * line and paragraph separators and NEL as they are, which is valid JSON and + * a line break to whatever splits text the way Unicode says. Written as + * escapes they are the same characters to a JSON parser. + */ +function lineOf(row: Record): string { + return JSON.stringify(row).replace( + /[\u0085\u2028\u2029]/g, + (character) => + `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`, + ); +} + /** Rows as a file: one to a line, so that two exports can be compared. */ function fileOf(rows: readonly Record[]): string { - return rows.length === 0 - ? '[]\n' - : `[\n${rows.map((row) => JSON.stringify(row)).join(',\n')}\n]\n`; + return rows.length === 0 ? '[]\n' : `[\n${rows.map(lineOf).join(',\n')}\n]\n`; +} + +/** Why the shop's part of an export was refused. */ +export class ExportTooLargeError extends Error {} + +/** + * Refuses an export, and says so in the Worker's log as well: Mallok's + * command line and admin show which plugin failed and not why. + */ +function refuse(table: string, what: string): never { + console.error( + JSON.stringify({ event: 'shop_export_refused', table, reason: what }), + ); + throw new ExportTooLargeError( + `The shop's ${table} ${what}, which is more than one export can carry. Nothing of the shop's was exported, so that a file missing rows is not taken for a backup.`, + ); } /** * The shop's files for a site export, read in one round trip. * - * Throws, naming the table, when one has more rows than an export carries. + * Throws, naming the table, when one has more rows than an export carries + * or the files together are more text than one does. */ export async function exportShopFiles( db: D1Database, options: { readonly defaultLocale: string; + /** The shop's settings, as Mallok hands them to the plugin. */ + readonly settings?: Readonly>; readonly rowLimit?: number; + readonly byteLimit?: number; }, ): Promise { const limit = options.rowLimit ?? EXPORT_ROW_LIMIT; + const byteLimit = options.byteLimit ?? EXPORT_BYTE_LIMIT; // One more than the limit is asked for: it is how "too many" is told from // "exactly as many as there may be". const results = await db.batch>( @@ -163,12 +210,11 @@ export async function exportShopFiles( ); const counts: Record = {}; + let bytes = 0; const files = TABLES.map((entry, index): PluginExportFile => { const rows = results[index]?.results ?? []; if (rows.length > limit) { - throw new Error( - `The shop's ${entry.name} have more than ${limit} rows, which is more than one export can carry. Nothing of the shop's was exported, so that a file missing rows is not taken for a backup.`, - ); + refuse(entry.name, `have more than ${limit} rows`); } const kept = entry.without === undefined @@ -181,8 +227,14 @@ export async function exportShopFiles( ), ); const path = `${FOLDER}/${entry.name}.json`; + const text = fileOf(kept); + // Counted in UTF-16 units, which is what a string costs in memory. + bytes += text.length * 2; + if (bytes > byteLimit) { + refuse(entry.name, `bring the export past ${byteLimit} bytes of text`); + } counts[entry.name] = { file: path, rows: kept.length }; - return { path, text: fileOf(kept) }; + return { path, text }; }); return [ @@ -190,6 +242,10 @@ export async function exportShopFiles( path: `${FOLDER}/manifest.json`, text: `${JSON.stringify({ format: FORMAT, tables: counts }, null, 2)}\n`, }, + { + path: `${FOLDER}/settings.json`, + text: `${JSON.stringify(options.settings ?? {}, null, 2)}\n`, + }, ...files, ]; } diff --git a/test/shop/export.test.ts b/test/shop/export.test.ts index 3efbc9f..29c0189 100644 --- a/test/shop/export.test.ts +++ b/test/shop/export.test.ts @@ -1,5 +1,6 @@ -import { beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { + EXPORT_BYTE_LIMIT, EXPORT_ROW_LIMIT, EXPORTED_TABLES, exportShopFiles, @@ -20,33 +21,41 @@ import { let screw: TestProduct; let washer: TestProduct; -const AT = '2026-10-09T08:00:00.000Z'; +const EARLIER = '2026-10-08T08:00:00.000Z'; +const LATER = '2026-10-09T08:00:00.000Z'; + +type Row = Record; +type ExportedFile = { path: string; text: string }; /** A file of the export, read back as the rows it holds. */ -function rowsOf( - files: readonly { path: string; text: string }[], - name: string, -): Record[] { +function rowsOf(files: readonly ExportedFile[], name: string): Row[] { const file = files.find((entry) => entry.path === `shop/${name}.json`); if (file === undefined) { throw new Error(`The export has no shop/${name}.json`); } - return JSON.parse(file.text) as Record[]; + return JSON.parse(file.text) as Row[]; } -function exported(rowLimit?: number) { - return exportShopFiles(db(), { - defaultLocale: 'en', - ...(rowLimit === undefined ? {} : { rowLimit }), - }); +function exported( + options: { + rowLimit?: number; + byteLimit?: number; + settings?: Record; + } = {}, +): Promise { + return exportShopFiles(db(), { defaultLocale: 'en', ...options }); } -async function table(sql: string): Promise[]> { - const { results } = await db().prepare(sql).all>(); +async function table(sql: string): Promise { + const { results } = await db().prepare(sql).all(); return results; } -/** One of everything the shop holds. */ +/** + * Two of everything the shop holds, and each pair put in the other way + * round from the order an export gives it — so that an export which merely + * read each table as it lay would be seen to. + */ async function fill(): Promise { await createVariant({ id: 'cs-16', @@ -70,6 +79,20 @@ async function fill(): Promise { productGroup: washer.translationGroup, sku: 'WA-5', }); + // A price whose variant is gone: no part of the shop writes one, and a + // seed or a statement typed by hand can. + await setPrice({ + variantId: 'ghost', + currency: 'USD', + amountMinor: 500, + source: 'base', + }); + await setPrice({ + variantId: 'cs-16', + currency: 'USD', + amountMinor: 210, + source: 'base', + }); await setPrice({ variantId: 'cs-10', currency: 'USD', @@ -83,77 +106,98 @@ async function fill(): Promise { source: 'auto', rateUsed: 0.92, }); - await setPrice({ - variantId: 'cs-16', - currency: 'USD', - amountMinor: 210, - source: 'base', - }); - await db().batch([ - db() - .prepare( - `INSERT INTO p_shop_stock_adjustment - (id, variant_id, delta, reason, ref_id, created_at) - VALUES ('manual:1', 'cs-10', -100, 'manual', NULL, ?), - ('order_paid:o-1:cs-10', 'cs-10', -100, 'order_paid', 'o-1', ?)`, - ) - .bind('2026-10-08T08:00:00.000Z', AT), + const order = (id: string, no: string, at: string) => db() .prepare( `INSERT INTO p_shop_order (id, order_no, status, currency, subtotal_minor, total_minor, stripe_payment_intent_id, email, shipping_address, locale, created_at, updated_at) - VALUES ('o-1', 'ND-261009-7K3M9QXA', 'paid', 'USD', 18500, 18500, - 'pi_1', 'buyer@order.example', ?, 'en', ?, ?)`, + VALUES (?, ?, 'paid', 'USD', 18500, 18500, ?, + 'buyer@order.example', ?, 'en', ?, ?)`, ) - .bind(JSON.stringify({ recipient: 'Ada', country: 'GB' }), AT, AT), + .bind( + id, + no, + `pi_${id}`, + JSON.stringify({ recipient: 'Ada', country: 'GB' }), + at, + at, + ); + await db().batch([ + db() + .prepare( + `INSERT INTO p_shop_stock_adjustment + (id, variant_id, delta, reason, ref_id, created_at) + VALUES ('order_paid:o-1:cs-10', 'cs-10', -100, 'order_paid', 'o-1', ?), + ('manual:1', 'cs-10', -100, 'manual', NULL, ?)`, + ) + .bind(LATER, EARLIER), + order('o-2', 'ND-261009-7K3M9QXB', LATER), + order('o-1', 'ND-261008-7K3M9QXA', EARLIER), db().prepare( `INSERT INTO p_shop_order_line (id, order_id, variant_id, sku_snapshot, name_snapshot, unit_price_minor, quantity) - VALUES ('o-1:cs-10', 'o-1', 'cs-10', 'CS-10', 'Cap screw M5', 185, 100)`, + VALUES ('o-1:cs-16', 'o-1', 'cs-16', 'CS-16', 'Cap screw M5', 210, 50), + ('o-1:cs-10', 'o-1', 'cs-10', 'CS-10', 'Cap screw M5', 185, 100)`, ), db() .prepare( `INSERT INTO p_shop_stripe_event (event_id, type, order_id, payment_intent_id, outcome, processed_at) - VALUES ('evt_1', 'payment_intent.succeeded', 'o-1', 'pi_1', 'paid', ?)`, + VALUES ('evt_2', 'payment_intent.succeeded', 'o-2', 'pi_o-2', 'paid', ?), + ('evt_1', 'payment_intent.succeeded', 'o-1', 'pi_o-1', 'paid', ?)`, ) - .bind(AT), + .bind(LATER, EARLIER), db() .prepare( `INSERT INTO p_shop_outbox (id, topic, order_id, ref, created_at) - VALUES ('order.paid:o-1', 'order.paid', 'o-1', NULL, ?)`, + VALUES ('order.paid:o-1', 'order.paid', 'o-1', NULL, ?), + ('payment.refused:evt_9', 'payment.refused', 'o-1', 'pi_9', ?)`, ) - .bind(AT), + .bind(EARLIER, LATER), // What an export leaves behind: a cart, a rate, the scheduler's state. db() .prepare( `INSERT INTO p_shop_cart (id, currency, locale, created_at, updated_at, expires_at) VALUES (?, 'USD', 'en', ?, ?, '2999-01-01')`, ) - .bind('a'.repeat(32), AT, AT), + .bind('a'.repeat(32), LATER, LATER), db() .prepare( "INSERT INTO p_shop_cart_line (cart_id, variant_id, quantity) VALUES (?, 'cs-10', 100)", ) .bind('a'.repeat(32)), + db() + .prepare( + `INSERT INTO p_shop_rate + (base_currency, quote_currency, rate, reference_date, fetched_at, source) + VALUES ('USD', 'EUR', 0.87654321, '2026-10-08', ?, 'ecb')`, + ) + .bind(LATER), db() .prepare( `INSERT INTO p_shop_state (key, value, updated_at) VALUES ('rates_tried_at', ?, ?)`, ) - .bind(AT, AT), + .bind(LATER, LATER), ]); + await storeInquiry({ + id: 'inq-2', + at: LATER, + form: { name: 'Grace Hopper', email: 'grace@buyer.example' }, + ipHash: 'e'.repeat(64), + }); await storeInquiry({ id: 'inq-1', - at: AT, + at: EARLIER, form: { name: 'Ada Lovelace', email: 'ada@buyer.example', company: '=Engines; Ltd', - message: 'Two lines,\nand a "quote".', + phone: '+44 20 7946 0000', + message: 'Two lines,\nand a "quote".\u2028A third, to Unicode.', }, ipHash: 'f'.repeat(64), }); @@ -205,16 +249,21 @@ describe('the shop in a site export', () => { 'shop/outbox.json', 'shop/payment-events.json', 'shop/prices.json', + 'shop/settings.json', 'shop/stock-adjustments.json', 'shop/variants.json', ]); - const variants = body.files.find( - (file) => file.path === 'shop/variants.json', - ); - expect(JSON.parse(variants?.text ?? '[]')).toHaveLength(3); + const text = (path: string): string => + body.files.find((file) => file.path === path)?.text ?? 'null'; + expect(JSON.parse(text('shop/variants.json'))).toHaveLength(3); + // The shop's own settings, which Mallok's `site.json` does not carry. + expect(JSON.parse(text('shop/settings.json'))).toMatchObject({ + rounding: 'ending99', + inquiry_acknowledge: false, + }); }); - it('carries every row of every table, column for column, as it is stored', async () => { + it('carries every row of every table, column for column, as it is stored and in a fixed order', async () => { await fill(); const files = await exported(); @@ -236,55 +285,90 @@ describe('the shop in a site export', () => { ], ] as const) { const stored = await table(sql); - expect(stored.length, name).toBeGreaterThan(0); + expect(stored.length, name).toBeGreaterThan(1); expect(rowsOf(files, name), name).toEqual(stored); } - // A variant, and with it what an export of the content calls its product. - expect(rowsOf(files, 'variants')).toEqual( + // And the order is the one meant, not the one the rows were put in by. + expect(rowsOf(files, 'stock-adjustments').map((row) => row.id)).toEqual([ + 'manual:1', + 'order_paid:o-1:cs-10', + ]); + expect(rowsOf(files, 'orders').map((row) => row.id)).toEqual([ + 'o-1', + 'o-2', + ]); + expect(rowsOf(files, 'order-lines').map((row) => row.id)).toEqual([ + 'o-1:cs-10', + 'o-1:cs-16', + ]); + expect(rowsOf(files, 'payment-events').map((row) => row.event_id)).toEqual([ + 'evt_1', + 'evt_2', + ]); + expect( + rowsOf(files, 'inquiry-lines').map( + (row) => `${row.inquiry_id}:${row.position}`, + ), + ).toEqual(['inq-1:0', 'inq-1:1', 'inq-2:0', 'inq-2:1']); + }); + + it('carries a variant whole, and with it what an export of the content calls its product', async () => { + await fill(); + + expect(rowsOf(await exported(), 'variants')).toEqual( (await table('SELECT * FROM p_shop_variant ORDER BY sku')).map((row) => ({ ...row, + // The default language's slug, or that of the one it is out in. product_slug: row.id === 'wa-5' ? 'export-scheibe' : 'export-screw', })), ); - // A price, and with it the SKU it is a price of. - expect(rowsOf(files, 'prices')).toEqual([ - expect.objectContaining({ - sku: 'CS-10', - currency: 'EUR', - amount_minor: 199, - source: 'auto', - rate_used: 0.92, - }), - expect.objectContaining({ - sku: 'CS-10', - currency: 'USD', - amount_minor: 185, - source: 'base', - rate_used: null, - }), - expect.objectContaining({ - sku: 'CS-16', - currency: 'USD', - amount_minor: 210, - }), - ]); }); - it('names a product by its slug in the default language, or in the one it is out in', async () => { + it('carries every price whole, the one of a variant that is gone as well', async () => { await fill(); + const skus: Record = { + 'cs-10': 'CS-10', + 'cs-16': 'CS-16', + ghost: null, + }; + + const prices = rowsOf(await exported(), 'prices'); - const bySku = Object.fromEntries( - rowsOf(await exported(), 'variants').map((row) => [ - row.sku, - row.product_slug, - ]), + expect(prices).toEqual( + ( + await table('SELECT * FROM p_shop_price ORDER BY variant_id, currency') + ).map((row) => ({ + ...row, + variant_sku: skus[String(row.variant_id)], + })), ); + expect(prices.map((row) => `${row.variant_id} ${row.currency}`)).toEqual([ + 'cs-10 EUR', + 'cs-10 USD', + 'cs-16 USD', + 'ghost USD', + ]); + }); - expect(bySku).toEqual({ - 'CS-10': 'export-screw', - 'CS-16': 'export-screw', - 'WA-5': 'export-scheibe', - }); + it('carries an inquiry whole but for the cart it came from and the mark of who sent it', async () => { + await fill(); + const files = await exported(); + const stored = await table( + 'SELECT * FROM p_shop_inquiry ORDER BY created_at, id', + ); + + expect(rowsOf(files, 'inquiries')).toEqual( + stored.map(({ cart_id: _cart, ip_hash: _mark, ...kept }) => kept), + ); + expect(rowsOf(files, 'inquiries').map((row) => row.id)).toEqual([ + 'inq-1', + 'inq-2', + ]); + const everything = files.map((file) => file.text).join('\n'); + for (const row of stored) { + expect(everything).not.toContain(String(row.cart_id)); + expect(everything).not.toContain(String(row.ip_hash)); + } }); it('keeps an amount the whole number it is, and what was typed as it was typed', async () => { @@ -298,26 +382,13 @@ describe('the shop in a site export', () => { }); // Nothing is rewritten for a spreadsheet's sake: this is for carrying. expect(rowsOf(files, 'inquiries')[0]).toMatchObject({ - name: 'Ada Lovelace', company: '=Engines; Ltd', - message: 'Two lines,\nand a "quote".', + phone: '+44 20 7946 0000', + message: 'Two lines,\nand a "quote".\u2028A third, to Unicode.', subtotal_minor: null, }); }); - it('leaves behind the cart an inquiry came from and the mark of who sent it', async () => { - await fill(); - const files = await exported(); - const [inquiry] = rowsOf(files, 'inquiries'); - const stored = await table('SELECT cart_id, ip_hash FROM p_shop_inquiry'); - - expect(inquiry).not.toHaveProperty('cart_id'); - expect(inquiry).not.toHaveProperty('ip_hash'); - const everything = files.map((file) => file.text).join('\n'); - expect(everything).not.toContain(String(stored[0]?.cart_id)); - expect(everything).not.toContain('f'.repeat(64)); - }); - it('leaves behind what only this deployment has a use for', async () => { await fill(); const files = await exported(); @@ -325,6 +396,8 @@ describe('the shop in a site export', () => { // A visitor's cart, and the id that is all that protects it. expect(everything).not.toContain('a'.repeat(32)); + // A rate, which is fetched again, and the scheduler's state. + expect(everything).not.toContain('0.87654321'); expect(everything).not.toContain('rates_tried_at'); expect(files.map((file) => file.path).join(' ')).not.toMatch( /cart|rate|state/, @@ -350,32 +423,70 @@ describe('the shop in a site export', () => { ); }); - it('says in a manifest what is in each file', async () => { + it('accounts for every column of every table it exports', async () => { + // A table is read with `*`, so that a column a migration adds is not + // forgotten. This is the other half: a new column has to be looked at + // here, and either listed or left behind in the export, before it is + // carried out of the shop without anyone having decided that it should. + const columns: Record = { + p_shop_variant: + 'id product_group sku option_values moq lead_time_min lead_time_max stock stock_policy weight_grams status sort_order created_at updated_at', + p_shop_price: + 'variant_id currency amount_minor source rate_used updated_at', + p_shop_stock_adjustment: 'id variant_id delta reason ref_id created_at', + p_shop_order: + 'id order_no status currency subtotal_minor shipping_minor tax_minor total_minor stripe_session_id stripe_payment_intent_id email shipping_address locale tracking_no created_at updated_at', + p_shop_order_line: + 'id order_id variant_id sku_snapshot name_snapshot unit_price_minor quantity', + p_shop_stripe_event: + 'event_id type order_id payment_intent_id outcome processed_at', + p_shop_outbox: 'seq id topic order_id ref created_at handled_at', + p_shop_inquiry: + 'id inquiry_no status name email company phone message locale currency country line_count subtotal_minor subtotal cart_id ip_hash notified_at acknowledged_at created_at updated_at', + p_shop_inquiry_line: + 'id inquiry_id position variant_id sku name quantity unit_price_minor unit_price line_total', + }; + + expect(Object.keys(columns).sort()).toEqual([...EXPORTED_TABLES].sort()); + for (const [name, expected] of Object.entries(columns)) { + const actual = ( + await table(`SELECT name FROM pragma_table_info('${name}')`) + ).map((row) => row.name); + expect(actual.join(' '), name).toBe(expected); + } + }); + + it('says in a manifest what is in each file, and carries the settings it is given', async () => { await fill(); - const files = await exported(); + const files = await exported({ settings: { rounding: 'integer' } }); expect(files[0]?.path).toBe('shop/manifest.json'); expect(JSON.parse(files[0]?.text ?? '{}')).toEqual({ format: 1, tables: { variants: { file: 'shop/variants.json', rows: 3 }, - prices: { file: 'shop/prices.json', rows: 3 }, + prices: { file: 'shop/prices.json', rows: 4 }, 'stock-adjustments': { file: 'shop/stock-adjustments.json', rows: 2 }, - orders: { file: 'shop/orders.json', rows: 1 }, - 'order-lines': { file: 'shop/order-lines.json', rows: 1 }, - 'payment-events': { file: 'shop/payment-events.json', rows: 1 }, - outbox: { file: 'shop/outbox.json', rows: 1 }, - inquiries: { file: 'shop/inquiries.json', rows: 1 }, - 'inquiry-lines': { file: 'shop/inquiry-lines.json', rows: 2 }, + orders: { file: 'shop/orders.json', rows: 2 }, + 'order-lines': { file: 'shop/order-lines.json', rows: 2 }, + 'payment-events': { file: 'shop/payment-events.json', rows: 2 }, + outbox: { file: 'shop/outbox.json', rows: 2 }, + inquiries: { file: 'shop/inquiries.json', rows: 2 }, + 'inquiry-lines': { file: 'shop/inquiry-lines.json', rows: 4 }, }, }); + expect(files[1]).toEqual({ + path: 'shop/settings.json', + text: '{\n "rounding": "integer"\n}\n', + }); }); it('exports an empty shop as files that are empty and valid', async () => { const files = await exported(); - expect(files).toHaveLength(10); - for (const file of files.slice(1)) { + expect(files).toHaveLength(11); + expect(files[1]?.text).toBe('{}\n'); + for (const file of files.slice(2)) { expect(file.text, file.path).toBe('[]\n'); } expect( @@ -389,17 +500,30 @@ describe('the shop in a site export', () => { ).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 0]); }); - it('is the same text for the same data, a row to a line', async () => { + it('writes a row to a line, to every reader of lines', async () => { await fill(); + const files = await exported(); + const file = (name: string): string => + files.find((entry) => entry.path === `shop/${name}.json`)?.text ?? ''; - const first = await exported(); - const second = await exported(); - - expect(second).toEqual(first); - const variants = first.find((file) => file.path === 'shop/variants.json'); // An opening line, a line for each of the three, a closing line. - expect(variants?.text.trimEnd().split('\n')).toHaveLength(5); - expect(variants?.text.endsWith(']\n')).toBe(true); + expect(file('variants').trimEnd().split('\n')).toHaveLength(5); + expect(file('variants').endsWith(']\n')).toBe(true); + // A message with a line separator in it is still one line to whatever + // splits text the way Unicode says, and the same message when parsed. + expect( + file('inquiries') + .trimEnd() + .split(/\r\n|[\n\v\f\r\u0085\u2028\u2029]/), + ).toHaveLength(4); + expect(file('inquiries')).toContain('\\u2028'); + expect(rowsOf(files, 'inquiries')[0]?.message).toContain('\u2028'); + }); + + it('is the same text for the same data', async () => { + await fill(); + + expect(await exported()).toEqual(await exported()); }); it('reads everything in one round trip', async () => { @@ -412,24 +536,140 @@ describe('the shop in a site export', () => { expect(calls).toBe(1); }); - it('fails, naming the table, rather than leave rows out', async () => { - await fill(); + describe('more than one export can carry', () => { + /** A database whose tables hold what a test says, by their place. */ + function holding(rows: readonly number[]): D1Database { + const statement = { bind: () => statement }; + return { + prepare: () => statement, + batch: async (statements: unknown[]) => + statements.map((_, index) => ({ + results: Array.from({ length: rows[index] ?? 0 }, (_unused, n) => ({ + id: `row-${n}`, + })), + })), + } as unknown as D1Database; + } - // Three variants: as many as there may be, and one more than that. - await expect(exported(3)).resolves.toHaveLength(10); - await expect(exported(2)).rejects.toThrow( - /^The shop's variants have more than 2 rows, which is more than one export can carry\./, + const NAMES = [ + 'variants', + 'prices', + 'stock-adjustments', + 'orders', + 'order-lines', + 'payment-events', + 'outbox', + 'inquiries', + 'inquiry-lines', + ]; + + it.each(NAMES.map((name, index) => [name, index] as const))( + 'fails, naming the %s, rather than leave rows out', + async (name, index) => { + const counts = (over: number): number[] => + NAMES.map((_, place) => (place === index ? over : 7)); + const options = { defaultLocale: 'en', rowLimit: 7 }; + + // As many as there may be, in every table. + await expect( + exportShopFiles(holding(counts(7)), options), + ).resolves.toHaveLength(11); + // One more than that, in this one. + await expect( + exportShopFiles(holding(counts(8)), options), + ).rejects.toThrow( + new RegExp( + `^The shop's ${name} have more than 7 rows, which is more than one export can carry\\.`, + ), + ); + }, ); - // And any other table the same: a fourth price, with three variants. - await setPrice({ - variantId: 'cs-16', - currency: 'EUR', - amountMinor: 199, - source: 'manual', + + it('asks each table for one row more than may be, which is how too many is told', async () => { + await fill(); + + // Three variants and four prices, really read. + await expect(exported({ rowLimit: 4 })).resolves.toHaveLength(11); + await expect(exported({ rowLimit: 3 })).rejects.toThrow( + /^The shop's prices have more than 3 rows/, + ); + await expect(exported({ rowLimit: 2 })).rejects.toThrow( + /^The shop's variants have more than 2 rows/, + ); + }); + + it('fails when the files together are more text than one export carries', async () => { + await fill(); + const size = (await exported()) + .slice(2) + .reduce((total, file) => total + file.text.length * 2, 0); + + await expect(exported({ byteLimit: size })).resolves.toHaveLength(11); + await expect(exported({ byteLimit: size - 1 })).rejects.toThrow( + /^The shop's inquiry-lines bring the export past \d+ bytes of text/, + ); + }); + + it('says why in the Worker’s log, with nothing of anyone’s in it', async () => { + await fill(); + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + await expect(exported({ rowLimit: 1 })).rejects.toThrow(); + expect(logged.mock.calls).toEqual([ + [ + '{"event":"shop_export_refused","table":"variants","reason":"have more than 1 rows"}', + ], + ]); + } finally { + logged.mockRestore(); + } + }); + + it('is reported to Mallok as the shop’s failure, and nothing of the shop’s is exported', async () => { + await db() + .prepare( + `WITH RECURSIVE n(i) AS ( + SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ? + ) + INSERT INTO p_shop_stock_adjustment + (id, variant_id, delta, reason, ref_id, created_at) + SELECT 'many:' || i, 'cs-10', 1, 'manual', NULL, ? FROM n`, + ) + .bind(EXPORT_ROW_LIMIT + 1, LATER) + .run(); + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}); + + let body: { + pluginExportFailures: { plugin: string; error: string }[]; + files: { path: string }[]; + }; + try { + body = await (await api('GET', '/_mallok/api/export')).json(); + } finally { + logged.mockRestore(); + } + + // Mallok's command line and admin refuse to call this a backup. + expect(body.pluginExportFailures).toEqual([ + { + plugin: 'shop', + error: expect.stringMatching( + new RegExp( + `^The shop's stock-adjustments have more than ${EXPORT_ROW_LIMIT} rows`, + ), + ), + }, + ]); + expect( + body.files.filter((file) => file.path.startsWith('shop/')), + ).toEqual([]); + }); + + it('has limits a Worker can be expected to hold', () => { + // Guards, not measurements: see the comment where they are set. + expect(EXPORT_ROW_LIMIT).toBe(5000); + expect(EXPORT_BYTE_LIMIT).toBe(16 * 1024 * 1024); }); - await expect(exported(3)).rejects.toThrow( - /^The shop's prices have more than 3 rows/, - ); - expect(EXPORT_ROW_LIMIT).toBeGreaterThanOrEqual(10_000); }); }); From 8d8d620c63c3f547c5673766bc317cab176be510 Mon Sep 17 00:00:00 2001 From: JasonYv Date: Fri, 9 Oct 2026 19:28:46 +0800 Subject: [PATCH 5/5] docs: record what the review of the export found, and correct what it showed wrong Removing the status condition does turn a race red where two different changes of an order meet; the design said it turned none. The security notes say what an export holds in full, and not to switch the shop off to get a backup when its export fails. --- CLAUDE.md | 4 ++-- README.md | 2 +- README.zh-CN.md | 2 +- SECURITY.md | 20 ++++++++++++------ .../2026-09-30-nundar-on-mallok-design.md | 21 +++++++++++++++---- 5 files changed, 35 insertions(+), 14 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 78aa6f0..11b0271 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Mallok decides the contracts on both sides. Its documentation is the reference: - **An inquiry's emails are a job** (`lib/inquiry-jobs.ts`), queued with `ctx.enqueueStatement` in that batch and given an id and nothing else. It may run twice: each email is marked on the inquiry once it has been handed to Mallok (`notified_at`, `acknowledged_at`), and only what is not marked is sent. The seller's is in English and answered to the buyer; the buyer's, in their language, is sent only when `inquiry_acknowledge` is on — it is off by default because the form has no challenge in front of it. This is the one place the plugin uses Mallok's jobs; the order code still writes its own outbox, which nothing drains. - **Inquiries are read in the admin** in a `table` panel with their lines as a related table, and marked, exported and deleted by the actions in `lib/inquiry-admin.ts`. An amount is stored twice, as minor units and as the text the buyer saw, because a table panel lists a column as it is stored. `cart_id` and `ip_hash` are in the table and not in the panel. - **An inquiry is the first personal data the shop holds.** It goes in no log and no address, and the plugin's own job carries an id only. Every value goes through `escapeHtml` on its way into an email; a name, a company and a phone number are made one line each (white space collapses; it is not refused, because a refused form comes back empty); an address is held to an allow-list (`EMAIL_PATTERN_SOURCE`), because the admin makes a `mailto:` link of it; a cell of the export that a spreadsheet would run as a formula is made text, and so is every piece a semicolon would make a cell of (`inert`). **What the plugin cannot keep to itself is the email**: `ctx.sendEmail` stores the whole message in Mallok's `job` table, where deleting the inquiry does not reach it. `SECURITY.md` says so; do not claim otherwise. -- **What the shop holds is in a site export** (`exportFiles`, `lib/export.ts`; design §19): variants with the slug of their product, prices, the stock ledger, orders and their lines, payment events, the outbox, inquiries and their lines — a JSON file each under `shop/`, a row an object, as stored, with a manifest. Tables are read with `*`, so a column a migration adds is exported without anyone remembering; `test/shop/export.test.ts` fails when a new `p_shop_` table is neither exported nor on its list of those left behind (carts, rates, the scheduler's state). An inquiry's `cart_id` and `ip_hash` stay behind. Nothing is rewritten for a spreadsheet: that is the admin's CSV, which is for reading; this is for carrying. A table over `EXPORT_ROW_LIMIT` fails the export by name rather than leave rows out, because Mallok then refuses to call it a backup — a file missing rows would pass for one. +- **What the shop holds is in a site export** (`exportFiles`, `lib/export.ts`; design §19): variants with the slug of their product, prices, the stock ledger, orders and their lines, payment events, the outbox, inquiries and their lines — a JSON file each under `shop/`, a row an object, as stored, with a manifest and the shop's own settings, which Mallok's `site.json` does not carry. Tables are read with `*`, so a column a migration adds is exported without anyone remembering; `test/shop/export.test.ts` fails when a new `p_shop_` table is neither exported nor on its list of those left behind (carts, rates, the scheduler's state), and when an exported table has a column the test does not list — so that a new column is looked at before it is carried out of the shop. A column an export adds to a row has a name no table's column has (`variant_sku`, `product_slug`): of two columns of one name D1 keeps the last. An inquiry's `cart_id` and `ip_hash` stay behind. Nothing is rewritten for a spreadsheet: that is the admin's CSV, which is for reading; this is for carrying. A table over `EXPORT_ROW_LIMIT`, or files over `EXPORT_BYTE_LIMIT` together, fail the export rather than leave rows out, because Mallok then refuses to call it a backup — a file missing rows would pass for one. Both are guards against a Worker's memory and neither is a measurement. The reason is in the endpoint's answer and in the Worker's log (`shop_export_refused`); Mallok's command line and admin show only that the shop failed, and suggest switching the failing plugin off — which would give a backup with nothing of the shop's in it. - **Orders and payment are built but not reachable** (design §14). `lib/orders.ts` places an order and confirms its payment, `lib/order-fulfilment.ts` ships, cancels and refunds, `lib/outbox.ts` records what each change still owes, `lib/stripe-webhook.ts` decides what a Stripe delivery means and which status to answer, `lib/stripe-signature.ts` and `lib/stripe-client.ts` talk to Stripe over `fetch`, `lib/order-email.ts` builds the buyer's emails. No route, page or admin screen calls them, and nothing drains the outbox: those wait for Mallok. Do not add a route that works around that. - **A change to an order is conditional on the status it was read in** (`ORDER_STILL_IN_STATUS` in `lib/order-guard.ts`), and the statement that changes the status comes last in its batch. That is what makes a racing second call write nothing. - **A change to an order writes its outbox row in the same batch** (`orderChangedOutbox`). What follows the change — an email, a purge — is owed from that row. Never send from a function's return value: it is lost whenever the Worker stops after the batch. Rows are read back by `seq`, the order they were committed in, never by `created_at`, which is whatever time the caller passed. @@ -115,7 +115,7 @@ When one is built, take it off this list and prove the behaviour with a test or - Tests in `test/shop` and `test/theme` run inside workerd against the site's own Worker. `test/shop/helpers.ts` brings a site up through Mallok's HTTP API (first request → admin → token → settings → plugin enabled); tables are created by Mallok's migrator, never by hand. Create content with `createContent`, not by inserting rows. - Mallok caches pages in `caches.default`. In a test, create all content before requesting any page. - A test for a fix must be seen failing without the fix. For new guards, break the guard and confirm the test goes red. -- A race is tested by running the calls with `Promise.all`, and such a test only counts once breaking the guard turns it red **every time**: that is the proof the two calls really interleave. Through `SELF.fetch` they do so about half the time, and where a request reads before it writes, the other half never reaches the guard. `holdWrites` in `test/shop/helpers.ts` holds each request at its write until the test has seen them all arrive; the inquiry's races use it. It waits by polling a flag: a promise settled by one request for another stops the Worker. Functions called side by side in one test are the other case: each runs as far as its reading before the next begins, every time. `atTheSameMoment` runs them and fails unless every call had read before any wrote, so that this is seen on every run and not assumed; the payment and fulfilment races use it. An order's write is stopped by several keys at once — the status it was read in, the event's keys, the ledger row's id, the outbox row's id — so taking one away turns no test red; taking them all away turns each race red every time. +- A race is tested by running the calls with `Promise.all`, and such a test only counts once breaking the guard turns it red **every time**: that is the proof the two calls really interleave. Through `SELF.fetch` they do so about half the time, and where a request reads before it writes, the other half never reaches the guard. `holdWrites` in `test/shop/helpers.ts` holds each request at its write until the test has seen them all arrive; the inquiry's races use it. It waits by polling a flag: a promise settled by one request for another stops the Worker. Functions called side by side in one test are the other case: each runs as far as its reading before the next begins, every time. `atTheSameMoment` runs them and fails unless every call had made its reading before any made its write, so that this is seen on every run and not assumed; the payment and fulfilment races use it. What it watches is the order of each call's trips to the database it was handed: tell it how many readings a function makes when that is not one, and have the function use that database and no other. A payment's write is stopped by several keys at once — the status it was read in, the event's keys, the ledger row's id, the outbox row's id — so taking the shared status condition away turns no race between two payments red; the races between two different changes of an order, which have no key in common, do go red. - `countD1Calls` in `test/shop/helpers.ts` counts round trips; use it wherever the number is a design constraint. `interceptBatches` runs a hook around each batch: it is how a test changes the data between a function's reading and its writing, or loses the answer to a write that committed. - `test/theme/pages.test.ts` renders every layout from content it creates itself, with everything set; `test/theme/bare.test.ts` does the same for a site that has filled in almost nothing, and fails on any empty element or `href=""`. Neither reads `content/`. A template that prints a wrapper has to check that there is something to put in it — and `content.html` is not a string, so capture it before comparing it with `blank`. The sample is checked by `test/content.test.ts` and `test/project.test.ts` (files only, no Worker) and by the smoke run. - `npm run smoke:shop` is the only place theme, plugin, content and the Mallok CLI run together; run it when touching any of them. It publishes the real sample, requests every kind of page, and follows every header and footer link in all four languages. diff --git a/README.md b/README.md index e2024a2..0168580 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Nundar builds on `mallok@0.1.0-rc.11`, the first release with the plugin API the | **Pricing** | USD base price; EUR and GBP derived from ECB rates with a buffer, rounding to a price point and a drift threshold; manual prices never overwritten; the pages of a product are purged from the cache when one of its prices moves | | | **Cart** | A form beside each size on a product page, and a cart page in the site's own design and language: set a quantity, remove a line, choose a currency. No script anywhere in it. MOQ, stock and the most a line may hold (500) are enforced by the server, which says what it refused on the cart page | Checkout | | **Inquiries** | The cart sent as one request for a quote, from a form on the cart page: who to answer, and a message. A size with no price can go in the cart to be asked about. The buyer lands on the inquiry's number; the seller is told by email, answerable straight to the buyer, and finds every inquiry in the admin with its lines — to mark, export as CSV, or delete | A challenge in front of the form (it waits for Mallok), which is why the email confirming to the buyer is off unless switched on | -| **Taking your data with you** | Everything the shop holds is in the site's export (`mallok export`), beside the content: variants, prices, the stock ledger, orders, payment events and inquiries, a JSON file each under `shop/`, as stored | An import of those files; an export of a table too large for one request (both wait for Mallok) | +| **Taking your data with you** | What the shop holds is in the site's export (`mallok export`), beside the content: variants, prices, the stock ledger, orders, payment events, inquiries and the shop's settings, a JSON file each under `shop/`, as stored | An import of those files; an export of a table too large for one request (both wait for Mallok) | | **Orders and payment** | The logic, tested and not yet reachable: orders with line snapshots, a payment that takes stock exactly once however often Stripe reports it, oversold orders, refunds that return stock, Stripe signature checks, order emails in four languages | The checkout and order pages, the webhook route, and order handling in the admin | The reasoning and the plan are in [`docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md`](docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md). diff --git a/README.zh-CN.md b/README.zh-CN.md index d9e5607..b6096aa 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -60,7 +60,7 @@ Nundar 基于 `mallok@0.1.0-rc.11`，这是第一个带有商城其余部分所 | **定价** | 美元基准价；欧元和英镑按 ECB 汇率换算，带缓冲、价位取整和漂移阈值；手动价格永不被覆盖；某个商品的价格变动时，清除该商品页面的缓存 | | | **购物车** | 商品页每个尺寸旁有一个表单；购物车页沿用站点自己的设计和语言，可改数量、移除、选币种，全程不需要脚本。起订量、库存和单行数量上限（500）由服务端校验，被拒绝的原因直接显示在购物车页上 | 结账 | | **询盘** | 在购物车页填一张表单（回复给谁、留言），把整个购物车作为一张询价单发出。没有价格的尺寸也可以加入购物车来询价。买家提交后看到询盘编号；卖家收到一封邮件，直接回复即可回到买家，并在后台看到每张询盘及其明细，可以标记、导出 CSV 或删除 | 表单前面的人机验证（等 Mallok 提供），因此给买家的确认邮件默认关闭，需要手动开启 | -| **带走你的数据** | 商城持有的全部数据都在站点导出（`mallok export`）里，和内容放在一起：规格、价格、库存台账、订单、付款事件和询盘，各为 `shop/` 下的一个 JSON 文件，按存储原样导出 | 把这些文件再导入；单次请求装不下的大表的导出（两者都要等 Mallok） | +| **带走你的数据** | 商城持有的数据都在站点导出（`mallok export`）里，和内容放在一起：规格、价格、库存台账、订单、付款事件、询盘和商城自己的设置，各为 `shop/` 下的一个 JSON 文件，按存储原样导出 | 把这些文件再导入；单次请求装不下的大表的导出（两者都要等 Mallok） | | **订单与付款** | 逻辑已写好并有测试，但还没有入口可以用到：带行快照的订单、无论 Stripe 通知多少次都只扣一次库存的付款、超卖订单、退还库存的退款、Stripe 签名校验、四种语言的订单邮件 | 结账页和订单页、webhook 路由、后台的订单处理 | 原因和计划见 [`docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md`](docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md)。 diff --git a/SECURITY.md b/SECURITY.md index 48fb19b..ebbc0a6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -46,7 +46,7 @@ security document describes each. Nundar implements none of them again. | Stripe keys and the webhook signing secret | Not stored yet | They will be plugin secrets, which Mallok encrypts. No code path reads them today. | | What a buyer says of themselves in an inquiry: name, email address, company, phone, message | D1, on the inquiry | Written when a cart is sent as a request for a quote. It is the first personal data the shop holds. Read in the admin by a signed-in administrator, or a token with `export`; deleted there. Never written to a log or put in an address, and the job the plugin queues carries an id only. **The email is another matter**: once handed to Mallok it sits, whole, in Mallok's own queue — see the residual risks. | | A mark of the visitor who sent an inquiry | D1, on the inquiry | Mallok's one-way hash of the connecting address, which is what the limit per visitor counts by. The address itself is never stored, and the admin's list does not show the mark. | -| The shop's part of a site export | Not stored: built when asked | Mallok's export, which needs a signed-in administrator or a token with `export`, includes every order and inquiry — names, email addresses, shipping addresses, messages — as files under `shop/`. Whoever holds an export holds that. It leaves out a cart's id and the mark of an inquiry's sender. | +| The shop's part of a site export | Not stored: built when asked | Mallok's export, which needs a signed-in administrator or a token with `export`, includes every order and inquiry — names, email addresses, phone numbers, companies, countries, shipping addresses, messages — as files under `shop/`, and the shop's settings, among them the address inquiries are announced to. Whoever holds an export holds that. It leaves out a cart's id and the mark of an inquiry's sender. | | Cart contents | D1, keyed by an unguessable 128-bit id | Variant ids and quantities only — **never prices**. A test asserts the table has no price column. | | The cart cookie | `HttpOnly`, `SameSite=Lax`, `Secure` on HTTPS, scoped to `/_mallok/p/shop` | It identifies a cart and nothing else. It is not a session and grants nothing in the admin. | | Prices, stock, variants | D1 | Changed only through the admin's variant form — which needs a signed-in administrator, or a token with `content:write` — or the plugin's own scheduled repricing. Every change of stock is a row in a ledger. | @@ -295,11 +295,19 @@ Stated plainly rather than left for an auditor to find: - **An export is as private as the database.** It carries orders and inquiries as they are stored. Keep an exported directory where the database's own backups would be kept, and do not commit one. -- **A shop can outgrow its export.** The export is built in one request; a - table with more rows than `EXPORT_ROW_LIMIT` fails it rather than be cut - short, and Mallok then produces no backup of the site at all. The limit is - a guard that has not been measured on a real account, and nothing prunes - the stock ledger or the orders. It is written up for Mallok. +- **A shop can outgrow its export.** The export is built in one request, in + memory; a table with more rows than `EXPORT_ROW_LIMIT`, or files of more + text together than `EXPORT_BYTE_LIMIT`, fail it rather than be cut short, + and Mallok then produces no backup of the site at all. The limits are + guards against a Worker's memory that have not been measured on a real + account — on the smallest plan the CPU allowance may stop an export well + before them — and nothing prunes the stock ledger or the orders. It is + written up for Mallok. +- **When the shop's export fails, do not switch the shop off to get a + backup.** Mallok's command line and admin say only that the plugin + failed, and suggest fixing or disabling it; with the shop disabled the + export succeeds and has nothing of the shop's in it. The reason is in the + Worker's log, under `shop_export_refused`. - **Sample content and sample variants are public test data.** Do not load `seed/shop-sample.sql` into a production database. The sample catalogue describes a supplier that does not exist: its certifications, test figures diff --git a/docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md b/docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md index 762b90b..bc2fc25 100644 --- a/docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md +++ b/docs/superpowers/specs/2026-09-30-nundar-on-mallok-design.md @@ -693,7 +693,7 @@ Not changed: the export still takes up to a thousand inquiries, as Mallok's own 4. *A job is queued beside a conditional write that stored nothing* in a true race, and finds nothing to send. Mallok's statement takes no condition. Written up. 5. *An amount is stored twice* because a table panel has no money column. Written up. 6. *Whether a quantity must be a multiple of the minimum order* (§17, left open, item 8) and *the rounding of a derived price* (item 3) were put to the owner with this work and are not settled in it. -7. *The older tests of a race* — a payment reported twice at once, two refunds, a payment against a cancellation, nine in all — were looked at the same day, after the review. They call functions side by side rather than send requests, and such calls do overlap every time: twenty-five rounds of two payments went to the database in one and the same order, both readings before either write. So none had to be rewritten for that. They now run through `atTheSameMoment`, which fails unless the calls overlapped, so that it is seen on every run rather than taken on trust; and each of the nine, with everything that stops its second write taken away, was red eight times in eight. One thing that showed: an order's write is stopped by several keys at once — the status condition, the event's two keys, the ledger row's id, the outbox row's id — and removing the status condition alone turns no payment test red. That is the design, not a gap in the tests. +7. *The older tests of a race* — a payment reported twice at once, two refunds, a payment against a cancellation, nine in all — were looked at the same day, after the review. They call functions side by side rather than send requests, and such calls do overlap every time: twenty-five rounds of two payments went to the database in one and the same order, both readings before either write. So none had to be rewritten for that. They now run through `atTheSameMoment`, which fails unless the calls overlapped, so that it is seen on every run rather than taken on trust; and each of the nine, with everything that stops its second write taken away, was red eight times in eight. One thing that showed: a payment's write is stopped by several keys at once — the status condition, the event's two keys, the ledger row's id, the outbox row's id — so that removing the shared status condition turns no race between two payments red. The races between two different changes of an order — shipping against refunding, paying against cancelling — have no key in common, and the status condition is all that stops them: those go red without it. `atTheSameMoment` watches the order of each call's trips to the database it was handed, and is told how many of them are readings; it refuses calls that never write, which would not be a race. 8. *A request with no connecting address.* Whether a zone that uses Cloudflare's "Remove visitor IP headers" transform still hands a Worker that header is not stated in the documentation that was read. If it does not, every visitor of such a site shares one limit of five an hour. ## 19. What the export changed or found (2026-10-09) @@ -709,15 +709,28 @@ A site export is how an operator leaves with what is theirs: `mallok export` wri 3. *The outbox and the payment events go too.* A row of the outbox that nobody has acted on is a duty — a refund that is owed — and losing it in a move is the worst thing an export could lose. 4. *Left behind*: carts, which are a visitor's and gone in a month; exchange rates, which are fetched again; the scheduler's state; and of an inquiry the cart it came from and the mark of who sent it. 5. *Every column, by `*`.* A column a later migration adds is exported without anyone remembering to list it. A test fails when a new `p_shop_` table is neither exported nor named among those left behind. -6. *A table past twenty thousand rows fails the export, by name.* An export is built whole, in one request. Leaving the oldest rows out would produce a file that passes for a backup; failing makes Mallok say that there is none. The figure is a guard and not a measurement. +6. *A table past five thousand rows, or files past sixteen megabytes of text together, fail the export.* An export is built whole, in one request, in memory. Leaving the oldest rows out would produce a file that passes for a backup; failing makes Mallok say that there is none. The figures are guards against a Worker's memory and not measurements; the first figure here was twenty thousand, which the review below showed to be more than a Worker can hold. +7. *The shop's own settings go too*, as `shop/settings.json`: Mallok's `site.json` carries the site's settings and no plugin's. -**How it was verified.** Twelve tests inside workerd: the files through Mallok's own export endpoint with no plugin reported as failed; every table's rows compared with the table, column for column; the slug in the default language and outside it; what is left behind, searched for in the whole text; an empty shop; the same text for the same data; one round trip; the limit at its edge for the first table and for another. Thirteen deliberate breakages, twelve red at once and one that called for the second of those. The smoke run reads the shop's nine variants, twenty-seven prices and one inquiry out of the export of a real local Worker, beside the inquiry plugin's own file, and was seen to fail with the export unwired. +**How it was verified.** Twenty-eight tests inside workerd: the files through Mallok's own export endpoint with no plugin reported as failed; every table's rows compared with the table, column for column and in a stated order, from rows put in the other way round; the price of a variant that is gone; an inquiry whole but for the two columns that stay; the slug in the default language and outside it; what is left behind, searched for in the whole text; every column of every exported table accounted for; an empty shop; a row to a line for every reader of lines; one round trip; each of the nine tables at the edge of the limit; the limit of text; the refusal as Mallok reports it, with nothing of the shop's exported. Thirty-nine deliberate breakages in two rounds, before and after the review below. The smoke run reads the shop's nine variants, twenty-seven prices and one inquiry out of the export of a real local Worker, beside the inquiry plugin's own file, and was seen to fail with the export unwired. -**Not verified.** `mallok export` itself was not run against this site: the test and the smoke run read the endpoint it reads. Nothing was deployed, and what an export costs a Worker in CPU and memory is not known. +**What an independent review then found.** Twelve things, each checked here before it was acted on. + +1. *A price whose variant was gone was left out*, and the manifest's count with it: the prices were read through a join. Nothing in the shop leaves such a row; a seed or a statement typed by hand can. Every price is read now. +2. *The limit did not guard what it said it did.* Twenty thousand rows of each table came to between seventy-four and a hundred and seventy-five megabytes of answer, which is more than a Worker's memory; the request would have died of that long before the limit named a table. Five thousand rows, and sixteen megabytes of text, now. +3. *The reason for a refusal does not reach the operator.* Mallok's endpoint carries it; its command line and admin print the plugin's id alone and suggest disabling the plugin. The reason is logged by the shop now, the security notes say not to follow that suggestion, and it is written up. +4. *The helper for races checked less than it said*: the first trip of every call before the second of any, which is "every reading before any write" only for a function that reads once. It is told the number of readings now, and refuses calls that never write. +5. *Four wrong exports passed every test*: one that dropped columns of an inquiry, one that dropped columns of a price, one with no order to its rows, one that held only two tables to the limit. Each is caught now. +6. *This document said that removing the status condition turns no payment race red.* Paying against cancelling does go red. Corrected in §18. +7. *The settings were in no export*, though the README said everything was. They are exported. +8. *Smaller things*: a line separator inside a value broke "a row to a line" for a reader that splits lines as Unicode does; a call that threw at once left the others unwaited for; a column added to a row could be hidden by a later column of the same name; the security notes left the phone number, the company and the country out of what an export holds. All put right. An integer past two to the fifty-third is changed by D1 on its way out; no amount here comes near. + +**Not verified.** `mallok export` itself was not run against this site: the test and the smoke run read the endpoint it reads. Nothing was deployed, and what an export costs a Worker in CPU and memory is not known: the review's own figures were taken locally, where neither is limited, and put the smallest plan's CPU allowance at a few hundred rows a table. Whether the statements of one batch count once or each towards D1's fifty queries a request was not found stated; the export's nine are few either way. **Left open.** 1. *Nothing imports these files.* Mallok's import knows content; a plugin has no hook to be handed its own files back. An export without an import is half of taking one's data with one. Written up. 2. *A shop can outgrow its export* (choice 6), and takes the whole site's backup with it when it does. Mallok has no way for a plugin to hand over a table a page at a time. Written up. +4. *An operator is not told why the shop's export failed*, and is advised to switch the plugin off. Written up. 3. *Mallok's own inquiry plugin exports its newest thousand rows and no more*, without saying so — found by reading it, while deciding what this export should do past its limit. Written up.