From 0165321a6eb452e06db70a577ff8e9ac05beafe9 Mon Sep 17 00:00:00 2001 From: Rami Date: Thu, 22 Oct 2026 23:41:44 +0100 Subject: [PATCH 1/2] refactor(ledger): the ledger keeps its layout number in a table on every store The layout number was kept two ways, in PRAGMA user_version on SQLite and in the table ledger_layout on PostgreSQL: one fact, two code paths and two sets of words. A pragma is also a property of a file, not of a schema, so a store that is not a file has none. Both stores now keep it in ledger_layout, through the path PostgreSQL took: requireTheLayoutKept lists the tables of the schema, reads the row of ledger_layout when it is among them, and writes the table and its row into a database that holds none. All a store says for itself is where its tables are listed, sqlite_master or pg_tables in current_schema(). The behaviour suite's layout part proves the same refusals in the same words on both stores. On SQLite the step runs in a transaction of its own, which dumbo begins IMMEDIATE: of 20 pairs of pools that took it on one new file at once, one failed with "table ledger_layout already exists" statement by statement, and none in the transaction. The key of the one row is NOT NULL, since SQLite 3.53.4 let a second row in under a null key. The table is new on SQLite, so the layout is 2 and the pinned catalog holds it. A SQLite database made before is refused as one that holds tables but no layout number. The server's tests make a ledger of another version with aSQLiteFileInLayout of @on.auto/ledger/testing, which writes the layout as the ledger does. Co-Authored-By: Claude Fable 5.1 --- packages/ledger/README.md | 10 ++-- packages/ledger/src/layout/layout-dialects.ts | 52 ++----------------- packages/ledger/src/layout/layout-kept.ts | 43 +++++++++++++-- .../src/layout/ledger-layout-behaviour.ts | 9 ++-- .../ledger/src/layout/ledger-layout.test.ts | 6 ++- packages/ledger/src/layout/ledger-layout.ts | 2 +- .../src/layout/sqlite-file-in-layout.test.ts | 22 ++++++++ .../src/layout/sqlite-file-in-layout.ts | 6 +++ .../ledger/src/layout/sqlite-schema.test.ts | 52 +++++++++++++++++++ packages/ledger/src/layout/sqlite-schema.ts | 6 +-- .../src/postgresql/postgresql-schema.test.ts | 25 +++++---- packages/ledger/src/sqlite-event-store.ts | 2 +- packages/ledger/src/testing/index.ts | 1 + .../src/lifecycle/ledger-startup.test.ts | 4 +- .../local-runtime/terminal-control.test.ts | 8 +-- .../records/a-ledger-of-another-version.ts | 10 ++-- 16 files changed, 163 insertions(+), 95 deletions(-) create mode 100644 packages/ledger/src/layout/sqlite-file-in-layout.test.ts create mode 100644 packages/ledger/src/layout/sqlite-file-in-layout.ts create mode 100644 packages/ledger/src/layout/sqlite-schema.test.ts diff --git a/packages/ledger/README.md b/packages/ledger/README.md index b58b3a186..676008075 100644 --- a/packages/ledger/README.md +++ b/packages/ledger/README.md @@ -448,15 +448,15 @@ The database's user needs `CREATE` on the `public` schema at every start, not on ## The layout of the ledger -`ledgerLayout`, in `src/layout/ledger-layout.ts`, numbers the layout of every table and index the ledger makes itself: the playbook's indexes, the content tables and their indexes, `kept_secrets`, the table of each projection with its indexes, the outcomes of runs among them, and on PostgreSQL the table that keeps the number. It is 1, and is raised by hand with any change to one of them. The tables Emmett and its database library make, whose names begin with `emt_` and `dmb_`, are theirs and outside the layout. +`ledgerLayout`, in `src/layout/ledger-layout.ts`, numbers the layout of every table and index the ledger makes itself: the playbook's indexes, the content tables and their indexes, `kept_secrets`, the table of each projection with its indexes, the outcomes of runs among them, and `ledger_layout`, the table that keeps the number. It is 2, and is raised by hand with any change to one of them. The tables Emmett and its database library make, whose names begin with `emt_` and `dmb_`, are theirs and outside the layout. -The ledger writes the number once, when it makes a database, before any table: on SQLite in `PRAGMA user_version`, as the first step of the store's `migrate`, before Emmett's migration; on PostgreSQL in the one row of `ledger_layout`, `layout INTEGER NOT NULL` beside a key whose `CHECK` allows a single value, in `onBeforeSchemaCreated`, inside Emmett's migration transaction, right after `migrationLockTakenWithin` has the advisory lock (`beforeTheSchema` of `src/postgresql/postgresql-schema.ts`), so servers that start together agree and a refusal rolls everything back. Every open applies one rule, `requireTheLayoutKept` of `src/layout/layout-kept.ts`, through a dialect of each store, `sqliteLayout` and `postgresqlLayout` of `src/layout/layout-dialects.ts`, which reads the number and counts the tables of the schema, in one statement on SQLite, and on PostgreSQL in one that counts them and finds `ledger_layout` among them and, when it does, a second that reads its row: +The ledger keeps the number in the one row of `ledger_layout` on every store, `layout INTEGER NOT NULL` beside a key, `one_row BOOLEAN NOT NULL PRIMARY KEY DEFAULT TRUE CHECK (one_row)`, that allows a single value. The key is `NOT NULL` because SQLite lets a primary key other than an `INTEGER PRIMARY KEY` hold `NULL`: without it, a second row with a null key went in on SQLite 3.53.4. The ledger writes the number once, when it makes a database, before any table, in the same transaction as the statement that found the database new: on SQLite in one of its own, which dumbo begins `IMMEDIATE` (`@event-driven-io/dumbo` 0.13.0-beta.56, `dist/sqlite3.js` line 3017), as the first step of the store's `migrate`, before Emmett's migration (`sqliteLayoutKept` of `src/layout/sqlite-schema.ts`); on PostgreSQL in `onBeforeSchemaCreated`, inside Emmett's migration transaction, right after `migrationLockTakenWithin` has the advisory lock (`beforeTheSchema` of `src/postgresql/postgresql-schema.ts`). So the table never stands without its row, ledgers that open a new database together agree, and a refusal rolls everything back: of 20 pairs of SQLite pools that took the step on one new file at once, one failed with `table ledger_layout already exists` when it ran statement by statement, and none in the transaction. Every open applies one rule in one code path, `requireTheLayoutKept` of `src/layout/layout-kept.ts`: a statement lists the tables of the schema, the one thing each store says for itself, through `sqliteLayout` and `postgresqlLayout` of `src/layout/layout-dialects.ts`, and, when `ledger_layout` is among them, a second reads its row: - A database in the layout kept opens, and the tables and indexes it lacks are created, each looked up first. - A database without a number that holds no table, `sqlite_master` holding none on SQLite and `pg_tables` none in `current_schema()` on PostgreSQL, is new: it is given the number, and the ledger goes on. - A database in another layout, or one that holds tables but no number, is refused as a `LedgerLayoutRefused` of `@on.auto/ledger`, tagged `ledger_layout_refused`, with `found`, the layout found or none, and `kept`, the layout this server keeps, in words that say to delete the local database, since nothing stored before launch is carried over. The server, which knows where the ledger is kept, answers it with words of its own: where the ledger is, the layout it keeps and the one it found, what to do, and where to ask. -`src/layout/ledger-layout.test.ts` pins the layout. It opens a new ledger on SQLite with the run outcome mapping and the projections the tests use, reads from `sqlite_master` the name and the statement of every table and index whose name begins with neither `emt_`, `dmb_` nor `sqlite_`, each statement on one line, and compares them, sorted by name, and `ledgerLayout` with the catalog it keeps for layout 1, so a change to anything the ledger makes fails it, in words that say to raise `ledgerLayout` and pin the catalog again. +`src/layout/ledger-layout.test.ts` pins the layout. It opens a new ledger on SQLite with the run outcome mapping and the projections the tests use, reads from `sqlite_master` the name and the statement of every table and index whose name begins with neither `emt_`, `dmb_` nor `sqlite_`, each statement on one line, and compares them, sorted by name, and `ledgerLayout` with the catalog it keeps for layout 2, so a change to anything the ledger makes fails it, in words that say to raise `ledgerLayout` and pin the catalog again. ## PostgreSQL @@ -482,7 +482,7 @@ What every store must provide, for the ledger to run at all: - The event store's own operations, to read a stream or its tail after a version, append with an expected version, migrate and close. A command makes at most one append and needs no transaction of its own. - The read of what a playbook recorded, with the indexes the ledger creates when it opens; each statement binds at most 15 parameters, and none needs a transaction. - An append on SQLite carries at most eight events: Emmett binds ten parameters for each event it inserts, and the ledger binds at most 100 in one statement there. -- A SQLite database must provide what the reads use: `octet_length` (SQLite 3.43 and later), window functions (3.25 and later), the JSON functions, of which the reads use `json_each` and `json_group_array` (built in since 3.38), and partial and expression indexes (3.8 and 3.9), what the rows of kept secrets use, an `UPDATE` and a `DELETE` with `RETURNING` (3.35), and what the layout of the ledger uses, `PRAGMA user_version` and its table-valued function `pragma_user_version` (3.16); `sqlite3` 6.0.1 carries SQLite 3.52.0. +- A SQLite database must provide what the reads use: `octet_length` (SQLite 3.43 and later), window functions (3.25 and later), the JSON functions, of which the reads use `json_each` and `json_group_array` (built in since 3.38), and partial and expression indexes (3.8 and 3.9), what the rows of kept secrets use, an `UPDATE` and a `DELETE` with `RETURNING` (3.35), and what the layout of the ledger uses, the literal `TRUE` (3.23); `sqlite3` 6.0.1 carries SQLite 3.52.0. - No adapter or application may nest a stream under `runs/` within a playbook: the key of a stream's kind ends at the fourth `/`, so the list of runs would take any stream named `runs//…` for a run. - Only the entries `src/sqlite3.ts` and `src/postgresql/postgresql-ledger.ts` know which driver is in use; the main entry loads neither `sqlite3` nor `pg`. @@ -496,7 +496,7 @@ Opened without the run outcome mapping and the projections, the ledger neither c ## Testing -The ledger's behaviour is one suite, in `src/testing/ledger-behaviour.ts`, that `src/ledger.test.ts` runs on SQLite and `src/postgresql/ledger-on-postgresql.test.ts` runs on PostgreSQL. Its part on the layout of the ledger, `src/layout/ledger-layout-behaviour.ts`, runs on both stores: a database that holds no table given the layout and opened again in it, a database in another layout refused with the layout found and the layout kept, in the words of the refusal, and left as it was, and a database that holds a table but no layout number refused the same way and left as it was; `src/layout/ledger-layout.test.ts` pins the layout on SQLite (see [The layout of the ledger](#the-layout-of-the-ledger)). Its part on the outcomes of runs, `src/outcomes/run-outcomes-behaviour.ts`, also runs on the in-memory ledger, in `src/outcomes/run-outcomes-in-memory.test.ts`, with the `runTallies` mapping of `@on.auto/operations/testing`; `src/outcomes/run-outcome-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a ledger opened without the projection, a table filled at open, an interrupted fill done again, and a table found and not filled again. That ledgers that start together fill the table once is tested on PostgreSQL alone, in `src/outcomes/run-outcomes-on-postgresql.test.ts`. That a server waits past the migrator's 10 s for a migration lock another server holds is tested with a fake lock and a fake clock in `src/postgresql/postgresql-run-outcomes.test.ts`, and on PostgreSQL in `src/postgresql/connections-on-postgresql.test.ts`, where a client holds the lock for 12 s. The part on the keyed projections, `src/projections/projections-behaviour.ts`, the rows kept, the reads by columns, in order and from where a page ended, the counts, the rows due across playbooks and the next due time, the rows a mapping keys over two stream kinds and the reader's advance, which a fold leaves as it stands unless it sets the columns, runs on both stores and on the in-memory ledger, in `src/projections/projections-in-memory.test.ts`, with the `runTallyRows` and `topicRows` projections of `@on.auto/operations/testing`, and `src/projections/projection-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a table made with its indexes and filled at open with the version before it dropped, a table found and not filled again, and a table keyed by its mapping filled in the order its facts were appended, over more than one batch. `src/postgresql/postgresql-projections.test.ts` checks the statements of the reads, the advance, the ordered fill and the fold's upsert on PostgreSQL without a database, and `src/postgresql/advances-on-postgresql.test.ts`, on PostgreSQL alone, a fold that read a row before an advance committed. The server tests the mapping of `@on.auto/definitions` on both stores, through the ledger it composes. Its part on reading what a playbook recorded, `src/testing/recorded-behaviour.ts`, `src/testing/runs-behaviour.ts`, `src/run-filters/run-filters-behaviour.ts`, `src/lineage/lineage-behaviour.ts` and `src/heads/heads-behaviour.ts`, the ids, causes and correlations a read gives, the runs of one capability or name, the read by correlation, the read from inside a record, and the versions and the read of heads, also runs on the in-memory ledger of `@on.auto/operations`, in `src/recorded/recorded-in-memory.test.ts`, so the three agree. A read behind an append whose transaction is still open, the same read by correlation, the same read newest first, a list of runs oldest first behind such an append, a write open in another database that hides nothing, and a start that finds the indexes while an append is open are tested on PostgreSQL alone, since SQLite serialises appends. Before each read of what a playbook recorded, the PostgreSQL entry of the suite waits until every committed message of its own database is older than every write open on the server, polling every 20 ms for at most 10 s and failing the test past that; the tests that wait have a timeout of 30 s, above vitest's 5 s. Waiting for the ledger's own horizon is not enough when other tests write on the same server: a write in another database that began before a message and ends while a read runs is in doubt to that read, which then stays behind it and may answer nothing, as a probe that ended such a write during a read confirmed. The test of a write open in another database waits for every write but that one. The SQLite entry and the in-memory ledger do not wait. `src/testing` holds 15 files, the most a folder holds, so the suite of lineage opened `src/lineage`, the suite of heads `src/heads`, the suite of the runs of one definition `src/run-filters`, the suite of the context of each message, the place of each record and the plans of the reads by definition and by id `src/context`, and the suite of recorded content `src/content`, which runs the behaviour suite of the port, `recordedContentBehaviour` of `@on.auto/operations/testing`, on both stores, reading through a second ledger on the same database and counting the chunks no head names, and adds what only chunks in a database show: content of no bytes, 100 bytes and 5 MB kept in chunks of 1 MiB under a head written last and read back a chunk at a time, a read whose chunk is missing failing after the chunks before it, no last chunk handed over of bytes that are not their digest's, a read of a staged content failing when its run is unstaged during it, and the chunks no head names swept on the ledger's schedule and before a close (`unnamed-chunks-behaviour.ts`, with `sweepsWatched` of `watched-sweeps.ts`, which lets a test wait for a sweep or hold it); its order of writes, the chunks first and the head last, the removal of a put's own chunks when another head stands, and an unstage's record and then its one transaction, the heads first, `src/content/content-store.test.ts` checks without a database, and `src/content/postgresql-content.test.ts` the hex a chunk travels as and the transaction of an unstage on one PostgreSQL connection, rolled back when a delete fails. The suites of kept secrets, `rowsBehaviour` and `storeBehaviour` of `@on.auto/secrets/testing`, run beside the ledger's on both stores, in the same two files, through `src/secrets/kept-secrets-behaviour.ts`, each on a database whose ledger opened first: every claim on the rows, and every claim on the store over them with the ledger's keeper and a key in hand, each moment set with `TestClock`, as that package's README lists them under [Testing](../secrets/README.md#testing), and the rows opened on connections of their own on the database the ledger made, which on SQLite migrate it again, and which leave it in the layout the ledger wrote. `src/secrets/postgresql-secret-rows.test.ts` checks the rows on PostgreSQL without a database, through fake connections: their pool, a lost connection logged, and each statement whole with the values it binds, the insert, the read with the end of its open claim, the forget and the holding it answers, the replacement, the claim with its condition of no open claim, the read of a keeper's rows under a kid, and the re-holding with its condition on the version and the holding read; `src/postgresql/postgresql-schema.test.ts` checks that the schema step creates its table only when it is missing, and that the migration takes its lock and counts the tables of the schema, then writes the layout into a database that holds none, or reads it from one that holds `ledger_layout`, before anything else. The read of one event by its id, with its data, context and place, no page cursor, and nothing of another playbook, is part of the suite of lineage, so it runs on both stores and on the in-memory ledger. `src/postgresql-reads/postgresql-runs.test.ts` checks the statement of that read on PostgreSQL without a database. `src/signal/append-signal.test.ts` tests the signal of an append on SQLite. +The ledger's behaviour is one suite, in `src/testing/ledger-behaviour.ts`, that `src/ledger.test.ts` runs on SQLite and `src/postgresql/ledger-on-postgresql.test.ts` runs on PostgreSQL. Its part on the layout of the ledger, `src/layout/ledger-layout-behaviour.ts`, runs on both stores: a database that holds no table given the layout and opened again in it, a database in another layout refused with the layout found and the layout kept, in the words of the refusal, and left as it was, and a database that holds a table but no layout number refused the same way and left as it was; `src/layout/ledger-layout.test.ts` pins the layout on SQLite (see [The layout of the ledger](#the-layout-of-the-ledger)), `src/layout/sqlite-schema.test.ts` checks without a database that SQLite lists the tables and writes the layout in one transaction, and `src/layout/sqlite-file-in-layout.test.ts` that `aSQLiteFileInLayout` of `@on.auto/ledger/testing`, with which the server's tests make a ledger of another version, puts a new file in the layout it is given. Its part on the outcomes of runs, `src/outcomes/run-outcomes-behaviour.ts`, also runs on the in-memory ledger, in `src/outcomes/run-outcomes-in-memory.test.ts`, with the `runTallies` mapping of `@on.auto/operations/testing`; `src/outcomes/run-outcome-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a ledger opened without the projection, a table filled at open, an interrupted fill done again, and a table found and not filled again. That ledgers that start together fill the table once is tested on PostgreSQL alone, in `src/outcomes/run-outcomes-on-postgresql.test.ts`. That a server waits past the migrator's 10 s for a migration lock another server holds is tested with a fake lock and a fake clock in `src/postgresql/postgresql-run-outcomes.test.ts`, and on PostgreSQL in `src/postgresql/connections-on-postgresql.test.ts`, where a client holds the lock for 12 s. The part on the keyed projections, `src/projections/projections-behaviour.ts`, the rows kept, the reads by columns, in order and from where a page ended, the counts, the rows due across playbooks and the next due time, the rows a mapping keys over two stream kinds and the reader's advance, which a fold leaves as it stands unless it sets the columns, runs on both stores and on the in-memory ledger, in `src/projections/projections-in-memory.test.ts`, with the `runTallyRows` and `topicRows` projections of `@on.auto/operations/testing`, and `src/projections/projection-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a table made with its indexes and filled at open with the version before it dropped, a table found and not filled again, and a table keyed by its mapping filled in the order its facts were appended, over more than one batch. `src/postgresql/postgresql-projections.test.ts` checks the statements of the reads, the advance, the ordered fill and the fold's upsert on PostgreSQL without a database, and `src/postgresql/advances-on-postgresql.test.ts`, on PostgreSQL alone, a fold that read a row before an advance committed. The server tests the mapping of `@on.auto/definitions` on both stores, through the ledger it composes. Its part on reading what a playbook recorded, `src/testing/recorded-behaviour.ts`, `src/testing/runs-behaviour.ts`, `src/run-filters/run-filters-behaviour.ts`, `src/lineage/lineage-behaviour.ts` and `src/heads/heads-behaviour.ts`, the ids, causes and correlations a read gives, the runs of one capability or name, the read by correlation, the read from inside a record, and the versions and the read of heads, also runs on the in-memory ledger of `@on.auto/operations`, in `src/recorded/recorded-in-memory.test.ts`, so the three agree. A read behind an append whose transaction is still open, the same read by correlation, the same read newest first, a list of runs oldest first behind such an append, a write open in another database that hides nothing, and a start that finds the indexes while an append is open are tested on PostgreSQL alone, since SQLite serialises appends. Before each read of what a playbook recorded, the PostgreSQL entry of the suite waits until every committed message of its own database is older than every write open on the server, polling every 20 ms for at most 10 s and failing the test past that; the tests that wait have a timeout of 30 s, above vitest's 5 s. Waiting for the ledger's own horizon is not enough when other tests write on the same server: a write in another database that began before a message and ends while a read runs is in doubt to that read, which then stays behind it and may answer nothing, as a probe that ended such a write during a read confirmed. The test of a write open in another database waits for every write but that one. The SQLite entry and the in-memory ledger do not wait. `src/testing` holds 15 files, the most a folder holds, so the suite of lineage opened `src/lineage`, the suite of heads `src/heads`, the suite of the runs of one definition `src/run-filters`, the suite of the context of each message, the place of each record and the plans of the reads by definition and by id `src/context`, and the suite of recorded content `src/content`, which runs the behaviour suite of the port, `recordedContentBehaviour` of `@on.auto/operations/testing`, on both stores, reading through a second ledger on the same database and counting the chunks no head names, and adds what only chunks in a database show: content of no bytes, 100 bytes and 5 MB kept in chunks of 1 MiB under a head written last and read back a chunk at a time, a read whose chunk is missing failing after the chunks before it, no last chunk handed over of bytes that are not their digest's, a read of a staged content failing when its run is unstaged during it, and the chunks no head names swept on the ledger's schedule and before a close (`unnamed-chunks-behaviour.ts`, with `sweepsWatched` of `watched-sweeps.ts`, which lets a test wait for a sweep or hold it); its order of writes, the chunks first and the head last, the removal of a put's own chunks when another head stands, and an unstage's record and then its one transaction, the heads first, `src/content/content-store.test.ts` checks without a database, and `src/content/postgresql-content.test.ts` the hex a chunk travels as and the transaction of an unstage on one PostgreSQL connection, rolled back when a delete fails. The suites of kept secrets, `rowsBehaviour` and `storeBehaviour` of `@on.auto/secrets/testing`, run beside the ledger's on both stores, in the same two files, through `src/secrets/kept-secrets-behaviour.ts`, each on a database whose ledger opened first: every claim on the rows, and every claim on the store over them with the ledger's keeper and a key in hand, each moment set with `TestClock`, as that package's README lists them under [Testing](../secrets/README.md#testing), and the rows opened on connections of their own on the database the ledger made, which on SQLite migrate it again, and which leave it in the layout the ledger wrote. `src/secrets/postgresql-secret-rows.test.ts` checks the rows on PostgreSQL without a database, through fake connections: their pool, a lost connection logged, and each statement whole with the values it binds, the insert, the read with the end of its open claim, the forget and the holding it answers, the replacement, the claim with its condition of no open claim, the read of a keeper's rows under a kid, and the re-holding with its condition on the version and the holding read; `src/postgresql/postgresql-schema.test.ts` checks that the schema step creates its table only when it is missing, and that the migration takes its lock and lists the tables of the schema, then writes the layout into a database that holds none, or reads it from one that holds `ledger_layout`, before anything else. The read of one event by its id, with its data, context and place, no page cursor, and nothing of another playbook, is part of the suite of lineage, so it runs on both stores and on the in-memory ledger. `src/postgresql-reads/postgresql-runs.test.ts` checks the statement of that read on PostgreSQL without a database. `src/signal/append-signal.test.ts` tests the signal of an append on SQLite. ```bash docker run --detach --name ledger-pg-test --publish 127.0.0.1:19632:5432 --env POSTGRES_PASSWORD=ledger-test postgres:18.6-alpine diff --git a/packages/ledger/src/layout/layout-dialects.ts b/packages/ledger/src/layout/layout-dialects.ts index c9b026f98..013351665 100644 --- a/packages/ledger/src/layout/layout-dialects.ts +++ b/packages/ledger/src/layout/layout-dialects.ts @@ -1,59 +1,13 @@ import { SQL } from '@event-driven-io/dumbo'; -import { Schema } from 'effect'; - -import type { StatementExecutor } from '../event-store.ts'; - -export interface FoundLayout { - readonly layout: number | undefined; - readonly tables: number; -} export interface LayoutDialect { - readonly found: (execute: StatementExecutor) => Promise; - readonly written: (layout: number) => readonly SQL[]; + readonly tablesOfTheSchema: () => SQL; } -const noLayout = 0; - -const NumberedRow = Schema.Tuple([Schema.Struct({ tables: Schema.Number, layout: Schema.Number })]); - export const sqliteLayout: LayoutDialect = { - found: async (execute) => { - const { rows } = await execute.query( - SQL`SELECT (SELECT count(*) FROM sqlite_master WHERE type = 'table') AS tables, user_version AS layout FROM pragma_user_version`, - ); - const [{ tables, layout }] = Schema.decodeUnknownSync(NumberedRow)(rows); - return { layout: layout === noLayout ? undefined : layout, tables }; - }, - written: (layout) => [SQL`PRAGMA user_version = ${SQL.plain(String(layout))}`], + tablesOfTheSchema: () => SQL`SELECT name FROM sqlite_master WHERE type = 'table'`, }; -const layoutTable = 'ledger_layout'; - -const TablesRow = Schema.Tuple([Schema.Struct({ tables: Schema.Number, numbered: Schema.Boolean })]); - -const LayoutRow = Schema.Tuple([Schema.Struct({ layout: Schema.Number })]); - -async function layoutKeptIn(execute: StatementExecutor): Promise { - const { rows } = await execute.query(SQL`SELECT layout FROM ${SQL.identifier(layoutTable)}`); - const [{ layout }] = Schema.decodeUnknownSync(LayoutRow)(rows); - return layout; -} - export const postgresqlLayout: LayoutDialect = { - found: async (execute) => { - const { rows } = await execute.query( - SQL`SELECT count(*)::int AS tables, count(*) FILTER (WHERE tablename = ${layoutTable}) > 0 AS numbered - FROM pg_tables WHERE schemaname = current_schema()`, - ); - const [{ tables, numbered }] = Schema.decodeUnknownSync(TablesRow)(rows); - return { layout: numbered ? await layoutKeptIn(execute) : undefined, tables }; - }, - written: (layout) => [ - SQL`CREATE TABLE ${SQL.identifier(layoutTable)} ( - one_row BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (one_row), - layout INTEGER NOT NULL - )`, - SQL`INSERT INTO ${SQL.identifier(layoutTable)} (layout) VALUES (${layout})`, - ], + tablesOfTheSchema: () => SQL`SELECT tablename AS name FROM pg_tables WHERE schemaname = current_schema()`, }; diff --git a/packages/ledger/src/layout/layout-kept.ts b/packages/ledger/src/layout/layout-kept.ts index d578cc17a..dbfe0ed80 100644 --- a/packages/ledger/src/layout/layout-kept.ts +++ b/packages/ledger/src/layout/layout-kept.ts @@ -1,20 +1,57 @@ +import { SQL } from '@event-driven-io/dumbo'; +import { Schema } from 'effect'; + +import { namesFound } from '../content/content-tables.ts'; import type { StatementExecutor } from '../event-store.ts'; import { inTurn } from '../projections/inline-projection.ts'; -import type { FoundLayout, LayoutDialect } from './layout-dialects.ts'; +import type { LayoutDialect } from './layout-dialects.ts'; import { ledgerLayoutRefused } from './ledger-layout-refused.ts'; import { ledgerLayout } from './ledger-layout.ts'; +export interface FoundLayout { + readonly layout: number | undefined; + readonly tables: number; +} + +const layoutTable = 'ledger_layout'; + +const LayoutRow = Schema.Tuple([Schema.Struct({ layout: Schema.Number })]); + +async function layoutKeptIn(execute: StatementExecutor): Promise { + const { rows } = await execute.query(SQL`SELECT layout FROM ${SQL.identifier(layoutTable)}`); + const [{ layout }] = Schema.decodeUnknownSync(LayoutRow)(rows); + return layout; +} + +export async function layoutFound(execute: StatementExecutor, dialect: LayoutDialect): Promise { + const tables = await namesFound(execute, dialect.tablesOfTheSchema()); + return { layout: tables.includes(layoutTable) ? await layoutKeptIn(execute) : undefined, tables: tables.length }; +} + +export function layoutWritten(execute: StatementExecutor, layout: number): Promise { + return inTurn( + [ + SQL`CREATE TABLE ${SQL.identifier(layoutTable)} ( + one_row BOOLEAN NOT NULL PRIMARY KEY DEFAULT TRUE CHECK (one_row), + layout INTEGER NOT NULL + )`, + SQL`INSERT INTO ${SQL.identifier(layoutTable)} (layout) VALUES (${layout})`, + ], + (statement) => execute.command(statement), + ); +} + function isNew({ layout, tables }: FoundLayout): boolean { return layout === undefined && tables === 0; } export async function requireTheLayoutKept(execute: StatementExecutor, dialect: LayoutDialect): Promise { - const found = await dialect.found(execute); + const found = await layoutFound(execute, dialect); if (found.layout === ledgerLayout) { return; } if (!isNew(found)) { throw ledgerLayoutRefused({ found: found.layout, kept: ledgerLayout }); } - await inTurn(dialect.written(ledgerLayout), (statement) => execute.command(statement)); + await layoutWritten(execute, ledgerLayout); } diff --git a/packages/ledger/src/layout/ledger-layout-behaviour.ts b/packages/ledger/src/layout/ledger-layout-behaviour.ts index 41ff404b3..8cc365d88 100644 --- a/packages/ledger/src/layout/ledger-layout-behaviour.ts +++ b/packages/ledger/src/layout/ledger-layout-behaviour.ts @@ -1,8 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { inTurn } from '../projections/inline-projection.ts'; import { aLedger, type LedgerEntry } from '../testing/ledger-entry.ts'; -import type { FoundLayout } from './layout-dialects.ts'; +import { layoutFound, layoutWritten, type FoundLayout } from './layout-kept.ts'; import { LedgerLayoutRefused } from './ledger-layout-refused.ts'; import { ledgerLayout } from './ledger-layout.ts'; @@ -12,13 +11,11 @@ const whatToDo = 'Nothing stored before launch is carried over, so delete the local database and start the server again'; export function layoutIn(entry: LedgerEntry, database: string): Promise { - return entry.executedOn(database, (execute) => entry.layout.found(execute)); + return entry.executedOn(database, (execute) => layoutFound(execute, entry.layout)); } function stampedWith(entry: LedgerEntry, database: string, layout: number): Promise { - return entry.executedOn(database, (execute) => - inTurn(entry.layout.written(layout), (statement) => execute.command(statement)), - ); + return entry.executedOn(database, (execute) => layoutWritten(execute, layout)); } function refusalOf(entry: LedgerEntry, database: string): Promise { diff --git a/packages/ledger/src/layout/ledger-layout.test.ts b/packages/ledger/src/layout/ledger-layout.test.ts index b9c3c6906..772a187f4 100644 --- a/packages/ledger/src/layout/ledger-layout.test.ts +++ b/packages/ledger/src/layout/ledger-layout.test.ts @@ -9,7 +9,7 @@ import { temporaryDatabase } from '../testing/temporary-database.ts'; import { ledgerLayout } from './ledger-layout.ts'; const pinned = { - layout: 1, + layout: 2, catalog: [ { name: 'kept_secrets', @@ -23,6 +23,10 @@ const pinned = { name: 'ledger_first_messages_by_kind', sql: "CREATE INDEX ledger_first_messages_by_kind ON emt_messages (substr(stream_id, 1, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + instr(substr(stream_id, instr(stream_id, '/') + instr(substr(stream_id, instr(stream_id, '/') + 1), '/') + 1), '/') + 1), '/')), global_position) WHERE stream_position = 1", }, + { + name: 'ledger_layout', + sql: 'CREATE TABLE ledger_layout (one_row BOOLEAN NOT NULL PRIMARY KEY DEFAULT TRUE CHECK (one_row), layout INTEGER NOT NULL)', + }, { name: 'ledger_messages_by_id', sql: 'CREATE INDEX ledger_messages_by_id ON emt_messages (message_id)' }, { name: 'ledger_messages_by_playbook', diff --git a/packages/ledger/src/layout/ledger-layout.ts b/packages/ledger/src/layout/ledger-layout.ts index 202d2fb39..ee291a598 100644 --- a/packages/ledger/src/layout/ledger-layout.ts +++ b/packages/ledger/src/layout/ledger-layout.ts @@ -1 +1 @@ -export const ledgerLayout = 1; +export const ledgerLayout = 2; diff --git a/packages/ledger/src/layout/sqlite-file-in-layout.test.ts b/packages/ledger/src/layout/sqlite-file-in-layout.test.ts new file mode 100644 index 000000000..9200c227e --- /dev/null +++ b/packages/ledger/src/layout/sqlite-file-in-layout.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { executedOn } from '../recorded/sqlite-checks.ts'; +import { temporaryDatabase } from '../testing/temporary-database.ts'; +import { sqliteLayout } from './layout-dialects.ts'; +import { layoutFound } from './layout-kept.ts'; +import { ledgerLayout } from './ledger-layout.ts'; +import { aSQLiteFileInLayout } from './sqlite-file-in-layout.ts'; + +describe('a SQLite file put in a layout of the ledger', () => { + it('holds that layout in the table that keeps it, and no other table', async () => { + const { fileName, remove } = temporaryDatabase(); + onTestFinished(remove); + + await aSQLiteFileInLayout(fileName, ledgerLayout + 1); + + expect(await executedOn(fileName, (execute) => layoutFound(execute, sqliteLayout))).toEqual({ + layout: ledgerLayout + 1, + tables: 1, + }); + }); +}); diff --git a/packages/ledger/src/layout/sqlite-file-in-layout.ts b/packages/ledger/src/layout/sqlite-file-in-layout.ts new file mode 100644 index 000000000..1a57bb1b5 --- /dev/null +++ b/packages/ledger/src/layout/sqlite-file-in-layout.ts @@ -0,0 +1,6 @@ +import { executedOn } from '../recorded/sqlite-checks.ts'; +import { layoutWritten } from './layout-kept.ts'; + +export function aSQLiteFileInLayout(fileName: string, layout: number): Promise { + return executedOn(fileName, (execute) => layoutWritten(execute, layout)); +} diff --git a/packages/ledger/src/layout/sqlite-schema.test.ts b/packages/ledger/src/layout/sqlite-schema.test.ts new file mode 100644 index 000000000..28f0a02c1 --- /dev/null +++ b/packages/ledger/src/layout/sqlite-schema.test.ts @@ -0,0 +1,52 @@ +import { SQL } from '@event-driven-io/dumbo'; +import { sqliteFormatter } from '@event-driven-io/dumbo/sqlite'; +import { describe, expect, it } from 'vitest'; + +import type { StatementExecutor, TransactionalPool } from '../event-store.ts'; +import { ledgerLayout } from './ledger-layout.ts'; +import { sqliteLayoutKept } from './sqlite-schema.ts'; + +function described(sql: SQL): string { + return SQL.describe(sql, sqliteFormatter).replaceAll(/\s+/gu, ' ').trim(); +} + +function aPoolHolding(tables: readonly string[]): { readonly pool: TransactionalPool; readonly asked: string[] } { + const asked: string[] = []; + const recordedAs = (where: string): StatementExecutor => ({ + query: (sql) => { + asked.push(`${where}${described(sql)}`); + return Promise.resolve({ rows: tables.map((name) => ({ name })) }); + }, + command: (sql) => { + asked.push(`${where}${described(sql)}`); + return Promise.resolve(); + }, + }); + return { + asked, + pool: { + execute: recordedAs('outside the transaction: '), + withTransaction: async (handle) => { + asked.push('BEGIN'); + await handle({ execute: recordedAs('') }); + asked.push('COMMIT'); + }, + }, + }; +} + +describe('the layout of the ledger on SQLite, as the store migrates', () => { + it('lists the tables of the schema and writes the layout into a table of one row of a database that holds none, all in one transaction', async () => { + const { pool, asked } = aPoolHolding([]); + + await sqliteLayoutKept(pool); + + expect(asked).toEqual([ + 'BEGIN', + "SELECT name FROM sqlite_master WHERE type = 'table'", + 'CREATE TABLE ledger_layout ( one_row BOOLEAN NOT NULL PRIMARY KEY DEFAULT TRUE CHECK (one_row), layout INTEGER NOT NULL )', + `INSERT INTO ledger_layout (layout) VALUES (${ledgerLayout})`, + 'COMMIT', + ]); + }); +}); diff --git a/packages/ledger/src/layout/sqlite-schema.ts b/packages/ledger/src/layout/sqlite-schema.ts index 8ce469fec..0a041ffd3 100644 --- a/packages/ledger/src/layout/sqlite-schema.ts +++ b/packages/ledger/src/layout/sqlite-schema.ts @@ -1,13 +1,13 @@ import { sqliteContent } from '../content/content-dialects.ts'; import { createdContentTables } from '../content/content-tables.ts'; -import type { StatementExecutor } from '../event-store.ts'; +import { inTransactionOf, type StatementExecutor, type TransactionalPool } from '../event-store.ts'; import { createSQLitePlaybookIndexes } from '../recorded/sqlite-indexes.ts'; import { createdSecretTable } from '../secrets/kept-secrets.ts'; import { sqliteLayout } from './layout-dialects.ts'; import { requireTheLayoutKept } from './layout-kept.ts'; -export function sqliteLayoutKept(execute: StatementExecutor): Promise { - return requireTheLayoutKept(execute, sqliteLayout); +export function sqliteLayoutKept(pool: TransactionalPool): Promise { + return inTransactionOf(pool)((execute) => requireTheLayoutKept(execute, sqliteLayout)); } export async function sqliteSchemaCreated(execute: StatementExecutor): Promise { diff --git a/packages/ledger/src/postgresql/postgresql-schema.test.ts b/packages/ledger/src/postgresql/postgresql-schema.test.ts index e1d5c2e51..e21cdd0f6 100644 --- a/packages/ledger/src/postgresql/postgresql-schema.test.ts +++ b/packages/ledger/src/postgresql/postgresql-schema.test.ts @@ -70,15 +70,15 @@ describe("the ledger's own schema on PostgreSQL", () => { }); interface Held { - readonly tables: number; + readonly tables: readonly string[]; readonly layout?: number; } -function answerTo(statement: string, { tables, layout }: Held): unknown { +function rowsAnswering(statement: string, { tables, layout }: Held): readonly unknown[] { if (statement.includes('pg_try_advisory_xact_lock')) { - return { locked: true }; + return [{ locked: true }]; } - return statement.includes('pg_tables') ? { tables, numbered: layout !== undefined } : { layout }; + return statement.includes('pg_tables') ? tables.map((name) => ({ name })) : [{ layout }]; } function aDatabaseHolding(held: Held): { readonly execute: StatementExecutor; readonly asked: string[] } { @@ -89,7 +89,7 @@ function aDatabaseHolding(held: Held): { readonly execute: StatementExecutor; re query: (sql) => { const statement = described(sql); asked.push(statement); - return Promise.resolve({ rows: [answerTo(statement, held)] }); + return Promise.resolve({ rows: rowsAnswering(statement, held) }); }, command: (sql) => { asked.push(described(sql)); @@ -101,28 +101,27 @@ function aDatabaseHolding(held: Held): { readonly execute: StatementExecutor; re const lockTaken = `SELECT pg_try_advisory_xact_lock(${emmettsMigrationLock}) AS locked`; -const tablesCounted = - 'SELECT count(*)::int AS tables, count(*) FILTER (WHERE tablename = "ledger_layout") > 0 AS numbered FROM pg_tables WHERE schemaname = current_schema()'; +const tablesListed = 'SELECT tablename AS name FROM pg_tables WHERE schemaname = current_schema()'; describe('the layout of the ledger on PostgreSQL, as the migration begins', () => { - it('takes the migration lock, then counts the tables of the schema, and writes the layout into a table of one row before any other table of a database that holds none', async () => { - const { execute, asked } = aDatabaseHolding({ tables: 0 }); + it('takes the migration lock, then lists the tables of the schema, and writes the layout into a table of one row before any other table of a database that holds none', async () => { + const { execute, asked } = aDatabaseHolding({ tables: [] }); await beforeTheSchema({ execute }); expect(asked).toEqual([ lockTaken, - tablesCounted, - 'CREATE TABLE ledger_layout ( one_row BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (one_row), layout INTEGER NOT NULL )', + tablesListed, + 'CREATE TABLE ledger_layout ( one_row BOOLEAN NOT NULL PRIMARY KEY DEFAULT TRUE CHECK (one_row), layout INTEGER NOT NULL )', `INSERT INTO ledger_layout (layout) VALUES (${ledgerLayout})`, ]); }); it('reads the layout of a database that holds its table, and writes nothing when it is the layout kept', async () => { - const { execute, asked } = aDatabaseHolding({ tables: 12, layout: ledgerLayout }); + const { execute, asked } = aDatabaseHolding({ tables: ['emt_messages', 'ledger_layout'], layout: ledgerLayout }); await beforeTheSchema({ execute }); - expect(asked).toEqual([lockTaken, tablesCounted, 'SELECT layout FROM ledger_layout']); + expect(asked).toEqual([lockTaken, tablesListed, 'SELECT layout FROM ledger_layout']); }); }); diff --git a/packages/ledger/src/sqlite-event-store.ts b/packages/ledger/src/sqlite-event-store.ts index d3d39cda6..117403c2d 100644 --- a/packages/ledger/src/sqlite-event-store.ts +++ b/packages/ledger/src/sqlite-event-store.ts @@ -45,7 +45,7 @@ export function sqliteEventStore( }), readRunOutcomes: sqliteRunOutcomesOf(kept.runOutcomes, pool.execute), migrate: async () => { - await sqliteLayoutKept(pool.execute); + await sqliteLayoutKept(pool); await streams.migrate(); await sqliteSchemaCreated(pool.execute); await preparedOn(projections.prepare, pool); diff --git a/packages/ledger/src/testing/index.ts b/packages/ledger/src/testing/index.ts index d438b6132..98ec7b2e8 100644 --- a/packages/ledger/src/testing/index.ts +++ b/packages/ledger/src/testing/index.ts @@ -1,2 +1,3 @@ export { ledgerLayout } from '../layout/ledger-layout.ts'; +export { aSQLiteFileInLayout } from '../layout/sqlite-file-in-layout.ts'; export { unreachableDatabase, type UnreachableDatabase } from './unreachable-database.ts'; diff --git a/packages/server/src/lifecycle/ledger-startup.test.ts b/packages/server/src/lifecycle/ledger-startup.test.ts index 5cc2372f7..2711e41bd 100644 --- a/packages/server/src/lifecycle/ledger-startup.test.ts +++ b/packages/server/src/lifecycle/ledger-startup.test.ts @@ -76,7 +76,7 @@ describe('a ledger file made by another version of Auto Runtime', () => { it('stops the process with one line of stderr, its words alone, that says where the ledger is, what to do and where to ask, ending with the command that deletes it', async () => { const ledger = temporaryLedger(); onTestFinished(ledger.remove); - aLedgerOfAnotherVersionAt(ledger.fileName); + await aLedgerOfAnotherVersionAt(ledger.fileName); const file = ledger.fileName; const child = spawnServer(mainModule, { HOST: '127.0.0.1', PORT: '0', LEDGER_FILE: file }); @@ -91,7 +91,7 @@ describe('a ledger file made by another version of Auto Runtime', () => { it('says in the image which files to delete from the volume, with no command', async () => { const ledger = temporaryLedger(); onTestFinished(ledger.remove); - aLedgerOfAnotherVersionAt(ledger.fileName); + await aLedgerOfAnotherVersionAt(ledger.fileName); const child = spawnServer(mainModule, { HOST: '127.0.0.1', diff --git a/packages/server/src/local-runtime/terminal-control.test.ts b/packages/server/src/local-runtime/terminal-control.test.ts index ea100f6e6..708508188 100644 --- a/packages/server/src/local-runtime/terminal-control.test.ts +++ b/packages/server/src/local-runtime/terminal-control.test.ts @@ -92,17 +92,17 @@ describe('a runtime started in a terminal on a taken port', () => { }); }); -function aHomeWithALedgerOfAnotherVersion(): { readonly home: string; readonly ledger: string } { +async function aHomeWithALedgerOfAnotherVersion(): Promise<{ readonly home: string; readonly ledger: string }> { const home = homeOfTheTest(homePrefix); const ledger = join(home, '.auto-runtime', 'ledger.db'); mkdirSync(join(home, '.auto-runtime')); - aLedgerOfAnotherVersionAt(ledger); + await aLedgerOfAnotherVersionAt(ledger); return { home, ledger }; } describe('a runtime started in a terminal on a ledger made by another version', () => { it('says on stderr, in a notice of its own, where the ledger is, what to do and where to ask, and exits', async () => { - const { home, ledger } = aHomeWithALedgerOfAnotherVersion(); + const { home, ledger } = await aHomeWithALedgerOfAnotherVersion(); const runtime = spawnRuntimeCommand([], { home, port: await aPortNothingListensOn() }); @@ -113,7 +113,7 @@ describe('a runtime started in a terminal on a ledger made by another version', }); it('writes the notice into a file that stderr is sent to with no color or link code', async () => { - const { home, ledger } = aHomeWithALedgerOfAnotherVersion(); + const { home, ledger } = await aHomeWithALedgerOfAnotherVersion(); const written = join(home, 'stderr.txt'); const runtime = spawnRuntimeCommand([], { home, port: await aPortNothingListensOn(), stderrInto: written }); diff --git a/packages/server/src/testing/records/a-ledger-of-another-version.ts b/packages/server/src/testing/records/a-ledger-of-another-version.ts index f291ce19d..30e7966d2 100644 --- a/packages/server/src/testing/records/a-ledger-of-another-version.ts +++ b/packages/server/src/testing/records/a-ledger-of-another-version.ts @@ -1,11 +1,7 @@ -import { DatabaseSync } from 'node:sqlite'; - -import { ledgerLayout } from '@on.auto/ledger/testing'; +import { aSQLiteFileInLayout, ledgerLayout } from '@on.auto/ledger/testing'; export const anotherLayout = ledgerLayout + 1; -export function aLedgerOfAnotherVersionAt(file: string): void { - const database = new DatabaseSync(file); - database.exec(`PRAGMA user_version = ${anotherLayout}`); - database.close(); +export function aLedgerOfAnotherVersionAt(file: string): Promise { + return aSQLiteFileInLayout(file, anotherLayout); } From f9f24f2597b44fe0a9c850fe84907b958b2f5d15 Mon Sep 17 00:00:00 2001 From: Rami Date: Thu, 22 Oct 2026 23:47:00 +0100 Subject: [PATCH 2/2] refactor(ledger): a content read takes a chunk as a Uint8Array or an ArrayBuffer The bytes of a chunk were decoded as a Uint8Array alone, where a SQL driver may answer an ArrayBuffer, which the standard byte APIs take as the same bytes. The read of a chunk now takes either and hands on a Uint8Array view of its bytes, through viewOf, so the shape a driver chooses is not the ledger's concern. sqlite3 6.0.1 and pg 8.23.1 both answer a Node Buffer, a Uint8Array, as a probe read; a test without a database reads a content back from each shape. Co-Authored-By: Claude Fable 5.1 --- packages/ledger/README.md | 4 +- packages/ledger/src/content/content-places.ts | 11 +++++- .../ledger/src/content/content-store.test.ts | 37 +++++++++++++++++-- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/ledger/README.md b/packages/ledger/README.md index 676008075..5dd7a21cc 100644 --- a/packages/ledger/README.md +++ b/packages/ledger/README.md @@ -263,7 +263,7 @@ The ledger keeps content whole beside its messages rather than in them, since ev | `staged_content` | playbook key, run id, digest | the head of a content staged under a run | | `unstaged_runs` | playbook key, run id | the runs whose staging ended | -`content.put(playbook, { stream, mediaType, expected? })` gives the put an upload id of its own, a UUIDv7, and reads the stream through `takenInChunks` of `@on.auto/operations` into chunks of 1 MiB, digesting and measuring the bytes as they pass, writing each chunk under its upload once the next has filled, in a statement of its own and so on PostgreSQL in a transaction of its own, since every open write holds back the reads oldest first (see [Order, and the horizon on PostgreSQL](#order-and-the-horizon-on-postgresql)). At the end of the stream it knows the digest and the size, and `requireExpected` refuses a digest or a size other than the one `expected` names as `not_its_digest` or `not_its_size`, as a stream that fails is refused as `stream_failed`, each a `ContentRefused`. A content of one chunk is still in hand then, so when the playbook holds its digest the put writes nothing and answers the head it holds; a put whose expected digest the playbook holds reads its stream through the digest and writes nothing; otherwise the put writes its last chunk, then the head, naming its upload, with `ON CONFLICT DO NOTHING`, and reads the head back. The first head written for a digest stands: a put that finds the head of another upload, whether it raced another put of the digest or found the digest kept, deletes its own chunks and answers the head it found, its size and its first media type, so two puts of one digest never write over each other's chunks, a wrong put racing a right one cannot spoil it, and a second media type never replaces the first. A put that fails part way, refused, by its stream or by its store, deletes its own chunks too, and never chunks a head names. Chunks no head names can still remain: those of a process that stopped between a chunk and its head or before its delete, and on PostgreSQL a chunk whose insert was still on its way, on a connection of its own, when its put was interrupted and deleted its chunks. So the ledger sweeps them, on a schedule of its own that starts when it opens and stops when it closes: once at the start, without holding the open back, and then every hour, it deletes every chunk no head names whose upload began more than an hour before, by the time a UUIDv7 carries in its first 48 bits. The sweep is maintenance: a sweep that fails, the first one among them, is logged as an error and never stops the ledger, and the next runs an hour later. A close waits for a sweep under way to finish before it closes the store, so it never leaves the sweep's connection open or runs the sweep on a closed store. A ledger opened with a content store of its own, `content: { content, sweepContent }` among the options of `ledgerLayer` and `postgresqlLedgerLayer`, keeps its content there and sweeps that store on the same schedule, and its content tables stay empty; the server opens it so when `CONTENT_STORE_URL` names an object store, whose adapter, [`@on.auto/object-storage`](../object-storage/README.md), sweeps the uploads its puts left behind. A put that streams a request body ends within the 300 s Node gives a request, `server.requestTimeout` of the server's `createServer`, well within the hour. `content.get(playbook, sha256)` reads the head and answers `{ stream, bytes, mediaType }`, or nothing for a digest without a head. The stream reads one chunk of the head's upload each time it is pulled, so neither a put nor a get holds the whole content in memory, and digests what it reads through `verifiedStream` of `@on.auto/operations`, the check the object store's reads go through too: a chunk that is missing, or bytes that are not those the digest names, fail the stream as a defect, as the ledger fails on a fact that no longer decodes, and it hands over the last chunk only once the size and the digest are checked, so no reader takes every byte of a content that is not its digest's. The content is keyed by the playbook and the SHA-256 of its bytes, so the key is the content's own address; a text is kept as its UTF-8, so its digest is the SHA-256 of the text the fact names. The digest is Node's own, `createHash('sha256')`, through `sha256Digest` of `@on.auto/operations`. A chunk travels to the store as the hex of its bytes, through `unhex` on SQLite and `decode(…, 'hex')` on PostgreSQL, since the SQL builder binds a byte array as JSON. A fact carries a digest only when its content was kept, so a read never joins content to a fact that kept none. Nothing kept is deleted. +`content.put(playbook, { stream, mediaType, expected? })` gives the put an upload id of its own, a UUIDv7, and reads the stream through `takenInChunks` of `@on.auto/operations` into chunks of 1 MiB, digesting and measuring the bytes as they pass, writing each chunk under its upload once the next has filled, in a statement of its own and so on PostgreSQL in a transaction of its own, since every open write holds back the reads oldest first (see [Order, and the horizon on PostgreSQL](#order-and-the-horizon-on-postgresql)). At the end of the stream it knows the digest and the size, and `requireExpected` refuses a digest or a size other than the one `expected` names as `not_its_digest` or `not_its_size`, as a stream that fails is refused as `stream_failed`, each a `ContentRefused`. A content of one chunk is still in hand then, so when the playbook holds its digest the put writes nothing and answers the head it holds; a put whose expected digest the playbook holds reads its stream through the digest and writes nothing; otherwise the put writes its last chunk, then the head, naming its upload, with `ON CONFLICT DO NOTHING`, and reads the head back. The first head written for a digest stands: a put that finds the head of another upload, whether it raced another put of the digest or found the digest kept, deletes its own chunks and answers the head it found, its size and its first media type, so two puts of one digest never write over each other's chunks, a wrong put racing a right one cannot spoil it, and a second media type never replaces the first. A put that fails part way, refused, by its stream or by its store, deletes its own chunks too, and never chunks a head names. Chunks no head names can still remain: those of a process that stopped between a chunk and its head or before its delete, and on PostgreSQL a chunk whose insert was still on its way, on a connection of its own, when its put was interrupted and deleted its chunks. So the ledger sweeps them, on a schedule of its own that starts when it opens and stops when it closes: once at the start, without holding the open back, and then every hour, it deletes every chunk no head names whose upload began more than an hour before, by the time a UUIDv7 carries in its first 48 bits. The sweep is maintenance: a sweep that fails, the first one among them, is logged as an error and never stops the ledger, and the next runs an hour later. A close waits for a sweep under way to finish before it closes the store, so it never leaves the sweep's connection open or runs the sweep on a closed store. A ledger opened with a content store of its own, `content: { content, sweepContent }` among the options of `ledgerLayer` and `postgresqlLedgerLayer`, keeps its content there and sweeps that store on the same schedule, and its content tables stay empty; the server opens it so when `CONTENT_STORE_URL` names an object store, whose adapter, [`@on.auto/object-storage`](../object-storage/README.md), sweeps the uploads its puts left behind. A put that streams a request body ends within the 300 s Node gives a request, `server.requestTimeout` of the server's `createServer`, well within the hour. `content.get(playbook, sha256)` reads the head and answers `{ stream, bytes, mediaType }`, or nothing for a digest without a head. The stream reads one chunk of the head's upload each time it is pulled, so neither a put nor a get holds the whole content in memory, and digests what it reads through `verifiedStream` of `@on.auto/operations`, the check the object store's reads go through too: a chunk that is missing, or bytes that are not those the digest names, fail the stream as a defect, as the ledger fails on a fact that no longer decodes, and it hands over the last chunk only once the size and the digest are checked, so no reader takes every byte of a content that is not its digest's. The content is keyed by the playbook and the SHA-256 of its bytes, so the key is the content's own address; a text is kept as its UTF-8, so its digest is the SHA-256 of the text the fact names. The digest is Node's own, `createHash('sha256')`, through `sha256Digest` of `@on.auto/operations`. A chunk travels to the store as the hex of its bytes, through `unhex` on SQLite and `decode(…, 'hex')` on PostgreSQL, since the SQL builder binds a byte array as JSON. It comes back in the shape its driver answers, a `Uint8Array` or an `ArrayBuffer`, which the standard byte APIs take as the same bytes, and the read takes either through one function, `viewOf` in `src/content/content-places.ts`, which hands on a `Uint8Array` view of the bytes without copying them; `sqlite3` 6.0.1 and `pg` 8.23.1 both answer a Node `Buffer`, a `Uint8Array`. A fact carries a digest only when its content was kept, so a read never joins content to a fact that kept none. Nothing kept is deleted. `content.stage(playbook, run, content)`, `content.staged(playbook, run, sha256)` and `content.unstage(playbook, run)` keep, read and remove content staged under a run, by its id, with the rules of a put and a get for the uploads, the heads and the digest, so a server reads what another staged on the same database. `unstage` records the run in `unstaged_runs` and then, in one transaction, removes the run's heads and then its chunks. The record stays, so `unstaged_runs` grows by one small row, the playbook key and the run id, for each run whose staging ended, as many as the runs that stage content; no capability of this repository stages content yet. A stage under a run already unstaged is refused with `StagingEnded` and leaves its stream unread, and a stage looks again once its head is written; if the run was unstaged meanwhile it deletes its own head and chunks and is refused the same way. Since the unstage records the run before it deletes and the stage writes its head before it looks, either the stage sees the record or the unstage's deletes see the head, so a stage that races an unstage leaves nothing behind. A read of a staged content whose run is unstaged while it is read fails at the first chunk it finds gone, as a defect, in words that say so: `The content staged under has no chunk , as when its run is unstaged while it is read`. @@ -496,7 +496,7 @@ Opened without the run outcome mapping and the projections, the ledger neither c ## Testing -The ledger's behaviour is one suite, in `src/testing/ledger-behaviour.ts`, that `src/ledger.test.ts` runs on SQLite and `src/postgresql/ledger-on-postgresql.test.ts` runs on PostgreSQL. Its part on the layout of the ledger, `src/layout/ledger-layout-behaviour.ts`, runs on both stores: a database that holds no table given the layout and opened again in it, a database in another layout refused with the layout found and the layout kept, in the words of the refusal, and left as it was, and a database that holds a table but no layout number refused the same way and left as it was; `src/layout/ledger-layout.test.ts` pins the layout on SQLite (see [The layout of the ledger](#the-layout-of-the-ledger)), `src/layout/sqlite-schema.test.ts` checks without a database that SQLite lists the tables and writes the layout in one transaction, and `src/layout/sqlite-file-in-layout.test.ts` that `aSQLiteFileInLayout` of `@on.auto/ledger/testing`, with which the server's tests make a ledger of another version, puts a new file in the layout it is given. Its part on the outcomes of runs, `src/outcomes/run-outcomes-behaviour.ts`, also runs on the in-memory ledger, in `src/outcomes/run-outcomes-in-memory.test.ts`, with the `runTallies` mapping of `@on.auto/operations/testing`; `src/outcomes/run-outcome-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a ledger opened without the projection, a table filled at open, an interrupted fill done again, and a table found and not filled again. That ledgers that start together fill the table once is tested on PostgreSQL alone, in `src/outcomes/run-outcomes-on-postgresql.test.ts`. That a server waits past the migrator's 10 s for a migration lock another server holds is tested with a fake lock and a fake clock in `src/postgresql/postgresql-run-outcomes.test.ts`, and on PostgreSQL in `src/postgresql/connections-on-postgresql.test.ts`, where a client holds the lock for 12 s. The part on the keyed projections, `src/projections/projections-behaviour.ts`, the rows kept, the reads by columns, in order and from where a page ended, the counts, the rows due across playbooks and the next due time, the rows a mapping keys over two stream kinds and the reader's advance, which a fold leaves as it stands unless it sets the columns, runs on both stores and on the in-memory ledger, in `src/projections/projections-in-memory.test.ts`, with the `runTallyRows` and `topicRows` projections of `@on.auto/operations/testing`, and `src/projections/projection-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a table made with its indexes and filled at open with the version before it dropped, a table found and not filled again, and a table keyed by its mapping filled in the order its facts were appended, over more than one batch. `src/postgresql/postgresql-projections.test.ts` checks the statements of the reads, the advance, the ordered fill and the fold's upsert on PostgreSQL without a database, and `src/postgresql/advances-on-postgresql.test.ts`, on PostgreSQL alone, a fold that read a row before an advance committed. The server tests the mapping of `@on.auto/definitions` on both stores, through the ledger it composes. Its part on reading what a playbook recorded, `src/testing/recorded-behaviour.ts`, `src/testing/runs-behaviour.ts`, `src/run-filters/run-filters-behaviour.ts`, `src/lineage/lineage-behaviour.ts` and `src/heads/heads-behaviour.ts`, the ids, causes and correlations a read gives, the runs of one capability or name, the read by correlation, the read from inside a record, and the versions and the read of heads, also runs on the in-memory ledger of `@on.auto/operations`, in `src/recorded/recorded-in-memory.test.ts`, so the three agree. A read behind an append whose transaction is still open, the same read by correlation, the same read newest first, a list of runs oldest first behind such an append, a write open in another database that hides nothing, and a start that finds the indexes while an append is open are tested on PostgreSQL alone, since SQLite serialises appends. Before each read of what a playbook recorded, the PostgreSQL entry of the suite waits until every committed message of its own database is older than every write open on the server, polling every 20 ms for at most 10 s and failing the test past that; the tests that wait have a timeout of 30 s, above vitest's 5 s. Waiting for the ledger's own horizon is not enough when other tests write on the same server: a write in another database that began before a message and ends while a read runs is in doubt to that read, which then stays behind it and may answer nothing, as a probe that ended such a write during a read confirmed. The test of a write open in another database waits for every write but that one. The SQLite entry and the in-memory ledger do not wait. `src/testing` holds 15 files, the most a folder holds, so the suite of lineage opened `src/lineage`, the suite of heads `src/heads`, the suite of the runs of one definition `src/run-filters`, the suite of the context of each message, the place of each record and the plans of the reads by definition and by id `src/context`, and the suite of recorded content `src/content`, which runs the behaviour suite of the port, `recordedContentBehaviour` of `@on.auto/operations/testing`, on both stores, reading through a second ledger on the same database and counting the chunks no head names, and adds what only chunks in a database show: content of no bytes, 100 bytes and 5 MB kept in chunks of 1 MiB under a head written last and read back a chunk at a time, a read whose chunk is missing failing after the chunks before it, no last chunk handed over of bytes that are not their digest's, a read of a staged content failing when its run is unstaged during it, and the chunks no head names swept on the ledger's schedule and before a close (`unnamed-chunks-behaviour.ts`, with `sweepsWatched` of `watched-sweeps.ts`, which lets a test wait for a sweep or hold it); its order of writes, the chunks first and the head last, the removal of a put's own chunks when another head stands, and an unstage's record and then its one transaction, the heads first, `src/content/content-store.test.ts` checks without a database, and `src/content/postgresql-content.test.ts` the hex a chunk travels as and the transaction of an unstage on one PostgreSQL connection, rolled back when a delete fails. The suites of kept secrets, `rowsBehaviour` and `storeBehaviour` of `@on.auto/secrets/testing`, run beside the ledger's on both stores, in the same two files, through `src/secrets/kept-secrets-behaviour.ts`, each on a database whose ledger opened first: every claim on the rows, and every claim on the store over them with the ledger's keeper and a key in hand, each moment set with `TestClock`, as that package's README lists them under [Testing](../secrets/README.md#testing), and the rows opened on connections of their own on the database the ledger made, which on SQLite migrate it again, and which leave it in the layout the ledger wrote. `src/secrets/postgresql-secret-rows.test.ts` checks the rows on PostgreSQL without a database, through fake connections: their pool, a lost connection logged, and each statement whole with the values it binds, the insert, the read with the end of its open claim, the forget and the holding it answers, the replacement, the claim with its condition of no open claim, the read of a keeper's rows under a kid, and the re-holding with its condition on the version and the holding read; `src/postgresql/postgresql-schema.test.ts` checks that the schema step creates its table only when it is missing, and that the migration takes its lock and lists the tables of the schema, then writes the layout into a database that holds none, or reads it from one that holds `ledger_layout`, before anything else. The read of one event by its id, with its data, context and place, no page cursor, and nothing of another playbook, is part of the suite of lineage, so it runs on both stores and on the in-memory ledger. `src/postgresql-reads/postgresql-runs.test.ts` checks the statement of that read on PostgreSQL without a database. `src/signal/append-signal.test.ts` tests the signal of an append on SQLite. +The ledger's behaviour is one suite, in `src/testing/ledger-behaviour.ts`, that `src/ledger.test.ts` runs on SQLite and `src/postgresql/ledger-on-postgresql.test.ts` runs on PostgreSQL. Its part on the layout of the ledger, `src/layout/ledger-layout-behaviour.ts`, runs on both stores: a database that holds no table given the layout and opened again in it, a database in another layout refused with the layout found and the layout kept, in the words of the refusal, and left as it was, and a database that holds a table but no layout number refused the same way and left as it was; `src/layout/ledger-layout.test.ts` pins the layout on SQLite (see [The layout of the ledger](#the-layout-of-the-ledger)), `src/layout/sqlite-schema.test.ts` checks without a database that SQLite lists the tables and writes the layout in one transaction, and `src/layout/sqlite-file-in-layout.test.ts` that `aSQLiteFileInLayout` of `@on.auto/ledger/testing`, with which the server's tests make a ledger of another version, puts a new file in the layout it is given. Its part on the outcomes of runs, `src/outcomes/run-outcomes-behaviour.ts`, also runs on the in-memory ledger, in `src/outcomes/run-outcomes-in-memory.test.ts`, with the `runTallies` mapping of `@on.auto/operations/testing`; `src/outcomes/run-outcome-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a ledger opened without the projection, a table filled at open, an interrupted fill done again, and a table found and not filled again. That ledgers that start together fill the table once is tested on PostgreSQL alone, in `src/outcomes/run-outcomes-on-postgresql.test.ts`. That a server waits past the migrator's 10 s for a migration lock another server holds is tested with a fake lock and a fake clock in `src/postgresql/postgresql-run-outcomes.test.ts`, and on PostgreSQL in `src/postgresql/connections-on-postgresql.test.ts`, where a client holds the lock for 12 s. The part on the keyed projections, `src/projections/projections-behaviour.ts`, the rows kept, the reads by columns, in order and from where a page ended, the counts, the rows due across playbooks and the next due time, the rows a mapping keys over two stream kinds and the reader's advance, which a fold leaves as it stands unless it sets the columns, runs on both stores and on the in-memory ledger, in `src/projections/projections-in-memory.test.ts`, with the `runTallyRows` and `topicRows` projections of `@on.auto/operations/testing`, and `src/projections/projection-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a table made with its indexes and filled at open with the version before it dropped, a table found and not filled again, and a table keyed by its mapping filled in the order its facts were appended, over more than one batch. `src/postgresql/postgresql-projections.test.ts` checks the statements of the reads, the advance, the ordered fill and the fold's upsert on PostgreSQL without a database, and `src/postgresql/advances-on-postgresql.test.ts`, on PostgreSQL alone, a fold that read a row before an advance committed. The server tests the mapping of `@on.auto/definitions` on both stores, through the ledger it composes. Its part on reading what a playbook recorded, `src/testing/recorded-behaviour.ts`, `src/testing/runs-behaviour.ts`, `src/run-filters/run-filters-behaviour.ts`, `src/lineage/lineage-behaviour.ts` and `src/heads/heads-behaviour.ts`, the ids, causes and correlations a read gives, the runs of one capability or name, the read by correlation, the read from inside a record, and the versions and the read of heads, also runs on the in-memory ledger of `@on.auto/operations`, in `src/recorded/recorded-in-memory.test.ts`, so the three agree. A read behind an append whose transaction is still open, the same read by correlation, the same read newest first, a list of runs oldest first behind such an append, a write open in another database that hides nothing, and a start that finds the indexes while an append is open are tested on PostgreSQL alone, since SQLite serialises appends. Before each read of what a playbook recorded, the PostgreSQL entry of the suite waits until every committed message of its own database is older than every write open on the server, polling every 20 ms for at most 10 s and failing the test past that; the tests that wait have a timeout of 30 s, above vitest's 5 s. Waiting for the ledger's own horizon is not enough when other tests write on the same server: a write in another database that began before a message and ends while a read runs is in doubt to that read, which then stays behind it and may answer nothing, as a probe that ended such a write during a read confirmed. The test of a write open in another database waits for every write but that one. The SQLite entry and the in-memory ledger do not wait. `src/testing` holds 15 files, the most a folder holds, so the suite of lineage opened `src/lineage`, the suite of heads `src/heads`, the suite of the runs of one definition `src/run-filters`, the suite of the context of each message, the place of each record and the plans of the reads by definition and by id `src/context`, and the suite of recorded content `src/content`, which runs the behaviour suite of the port, `recordedContentBehaviour` of `@on.auto/operations/testing`, on both stores, reading through a second ledger on the same database and counting the chunks no head names, and adds what only chunks in a database show: content of no bytes, 100 bytes and 5 MB kept in chunks of 1 MiB under a head written last and read back a chunk at a time, a read whose chunk is missing failing after the chunks before it, no last chunk handed over of bytes that are not their digest's, a read of a staged content failing when its run is unstaged during it, and the chunks no head names swept on the ledger's schedule and before a close (`unnamed-chunks-behaviour.ts`, with `sweepsWatched` of `watched-sweeps.ts`, which lets a test wait for a sweep or hold it); its order of writes, the chunks first and the head last, the removal of a put's own chunks when another head stands, a read of a chunk its store answers as a `Uint8Array` or as an `ArrayBuffer`, and an unstage's record and then its one transaction, the heads first, `src/content/content-store.test.ts` checks without a database, and `src/content/postgresql-content.test.ts` the hex a chunk travels as and the transaction of an unstage on one PostgreSQL connection, rolled back when a delete fails. The suites of kept secrets, `rowsBehaviour` and `storeBehaviour` of `@on.auto/secrets/testing`, run beside the ledger's on both stores, in the same two files, through `src/secrets/kept-secrets-behaviour.ts`, each on a database whose ledger opened first: every claim on the rows, and every claim on the store over them with the ledger's keeper and a key in hand, each moment set with `TestClock`, as that package's README lists them under [Testing](../secrets/README.md#testing), and the rows opened on connections of their own on the database the ledger made, which on SQLite migrate it again, and which leave it in the layout the ledger wrote. `src/secrets/postgresql-secret-rows.test.ts` checks the rows on PostgreSQL without a database, through fake connections: their pool, a lost connection logged, and each statement whole with the values it binds, the insert, the read with the end of its open claim, the forget and the holding it answers, the replacement, the claim with its condition of no open claim, the read of a keeper's rows under a kid, and the re-holding with its condition on the version and the holding read; `src/postgresql/postgresql-schema.test.ts` checks that the schema step creates its table only when it is missing, and that the migration takes its lock and lists the tables of the schema, then writes the layout into a database that holds none, or reads it from one that holds `ledger_layout`, before anything else. The read of one event by its id, with its data, context and place, no page cursor, and nothing of another playbook, is part of the suite of lineage, so it runs on both stores and on the in-memory ledger. `src/postgresql-reads/postgresql-runs.test.ts` checks the statement of that read on PostgreSQL without a database. `src/signal/append-signal.test.ts` tests the signal of an append on SQLite. ```bash docker run --detach --name ledger-pg-test --publish 127.0.0.1:19632:5432 --env POSTGRES_PASSWORD=ledger-test postgres:18.6-alpine diff --git a/packages/ledger/src/content/content-places.ts b/packages/ledger/src/content/content-places.ts index 2a70d8430..3598a2144 100644 --- a/packages/ledger/src/content/content-places.ts +++ b/packages/ledger/src/content/content-places.ts @@ -29,7 +29,13 @@ const HeadRows = Schema.Array( Schema.Struct({ upload_id: Schema.String, bytes: Schema.Number, chunks: Schema.Number, media_type: Schema.String }), ); -const ChunkRows = Schema.Array(Schema.Struct({ data: Schema.Uint8Array })); +const StoredBytes = Schema.Union([Schema.Uint8Array, Schema.instanceOf(ArrayBuffer)]); + +const ChunkRows = Schema.Array(Schema.Struct({ data: StoredBytes })); + +function viewOf(stored: Readonly | Readonly): Uint8Array { + return stored instanceof Uint8Array ? stored : new Uint8Array(stored); +} export function matching(key: Key): SQL { return SQL.merge( @@ -84,7 +90,8 @@ async function chunkOf({ execute, shelf }: Shelved, upload: Key, index: number): const { rows } = await execute.query( SQL`SELECT data FROM ${SQL.identifier(shelf.chunks)} WHERE ${matching(upload)} AND chunk = ${index}`, ); - return Schema.decodeUnknownSync(ChunkRows)(rows).at(0)?.data; + const chunk = Schema.decodeUnknownSync(ChunkRows)(rows).at(0); + return chunk === undefined ? undefined : viewOf(chunk.data); } export function uploadOf(place: Key, uploadId: string): Key { diff --git a/packages/ledger/src/content/content-store.test.ts b/packages/ledger/src/content/content-store.test.ts index f86712fd8..2c424ef98 100644 --- a/packages/ledger/src/content/content-store.test.ts +++ b/packages/ledger/src/content/content-store.test.ts @@ -1,6 +1,6 @@ import { SQL } from '@event-driven-io/dumbo'; import { sqliteFormatter } from '@event-driven-io/dumbo/sqlite'; -import { mostContentChunkBytes, sha256Of, textToKeep } from '@on.auto/operations'; +import { mostContentChunkBytes, sha256Of, textOfContent, textToKeep } from '@on.auto/operations'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -23,12 +23,13 @@ interface Recording { readonly statements: readonly string[]; } -function recordingFinding(heads: readonly unknown[]): Recording { +function recordingAnswering(rowsOf: (statement: string) => readonly unknown[]): Recording { const statements: string[] = []; const execute: StatementExecutor = { query: (sql) => { - statements.push(described(sql)); - return Promise.resolve({ rows: heads }); + const statement = described(sql); + statements.push(statement); + return Promise.resolve({ rows: rowsOf(statement) }); }, command: (sql) => { statements.push(described(sql)); @@ -48,12 +49,27 @@ function recordingFinding(heads: readonly unknown[]): Recording { }; } +function recordingFinding(heads: readonly unknown[]): Recording { + return recordingAnswering(() => heads); +} + +function recordingHoldingOneChunk(data: unknown): Recording { + return recordingAnswering((statement) => (statement.startsWith('SELECT data') ? [{ data }] : [headOf('upload')])); +} + function kindsOf({ statements }: Recording): readonly string[] { return statements.map((statement) => statement.split(' ').slice(0, 3).join(' ')); } const largerThanTwoChunks = 'x'.repeat(2 * mostContentChunkBytes + 1); +const keptBytes = new TextEncoder().encode('kept'); + +const chunkShapes: ReadonlyArray = [ + ['a Uint8Array', keptBytes], + ['an ArrayBuffer', keptBytes.slice().buffer], +]; + function headOf(uploadId: string) { return { upload_id: uploadId, bytes: 4, chunks: 1, media_type: 'text/plain' }; } @@ -131,6 +147,19 @@ describe('the reads and the removals of a recorded content', () => { expect([read, kindsOf(recording)]).toEqual([undefined, ['SELECT upload_id, CAST(bytes']]); }); + it.each(chunkShapes)('read a content back from a chunk its store answers as %s', async (_shape, chunk) => { + const recording = recordingHoldingOneChunk(chunk); + + const read = await Effect.runPromise( + Effect.flatMap( + Effect.flatMap(sqliteContentOn(recording.pool).content.get(alpha, sha256Of('kept')), Effect.fromNullishOr), + textOfContent, + ), + ); + + expect(read).toBe('kept'); + }); + it('mark a run unstaged, then remove the heads and the chunks it staged in one transaction, the heads first', async () => { const recording = recordingFinding([]);