diff --git a/contrib/amcheck/verify_common.c b/contrib/amcheck/verify_common.c index 46e53b3f99ab6..2c31664b1bbc5 100644 --- a/contrib/amcheck/verify_common.c +++ b/contrib/amcheck/verify_common.c @@ -20,6 +20,7 @@ #include "catalog/pg_am.h" #include "commands/defrem.h" #include "commands/tablecmds.h" +#include "doltgres/hooks.h" #include "utils/guc.h" #include "utils/syscache.h" @@ -167,7 +168,7 @@ index_checkable(Relation rel, Oid am_id) errmsg("expected index as targets for verification"), errdetail_relkind_not_supported(rel->rd_rel->relkind))); - if (rel->rd_rel->relam != am_id) + if (rel->rd_rel->relam != am_id || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("expected \"%s\" index as targets for verification", get_am_name(am_id)), diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 4ecadc88d16cc..8d61bf7696950 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -21,6 +21,7 @@ #include "access/xact.h" #include "catalog/pg_am.h" #include "catalog/pg_class.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "storage/bufmgr.h" @@ -344,8 +345,8 @@ verify_heapam(PG_FUNCTION_ARGS) * Sequences always use heap AM, but they don't show that in the catalogs. * Other relkinds might be using a different AM, so check. */ - if (ctx.rel->rd_rel->relkind != RELKIND_SEQUENCE && - ctx.rel->rd_rel->relam != HEAP_TABLE_AM_OID) + if ((ctx.rel->rd_rel->relkind != RELKIND_SEQUENCE && + ctx.rel->rd_rel->relam != HEAP_TABLE_AM_OID) || doltgres_heap_only_refused(ctx.rel)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only heap AM is supported"))); diff --git a/contrib/pageinspect/brinfuncs.c b/contrib/pageinspect/brinfuncs.c index 309b9522f9022..8beeb46c9e7cc 100644 --- a/contrib/pageinspect/brinfuncs.c +++ b/contrib/pageinspect/brinfuncs.c @@ -15,6 +15,7 @@ #include "access/htup_details.h" #include "catalog/pg_am_d.h" #include "catalog/pg_type.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "lib/stringinfo.h" #include "miscadmin.h" @@ -165,7 +166,7 @@ brin_page_items(PG_FUNCTION_ARGS) indexRel = index_open(indexRelid, AccessShareLock); - if (!IS_BRIN(indexRel)) + if (!IS_BRIN(indexRel) || doltgres_heap_only_refused(indexRel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a %s index", diff --git a/contrib/pageinspect/btreefuncs.c b/contrib/pageinspect/btreefuncs.c index 62c905c6e7c2c..222a8677eb663 100644 --- a/contrib/pageinspect/btreefuncs.c +++ b/contrib/pageinspect/btreefuncs.c @@ -33,6 +33,7 @@ #include "catalog/namespace.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "pageinspect.h" @@ -225,7 +226,7 @@ check_relation_block_range(Relation rel, int64 blkno) static void bt_index_block_validate(Relation rel, int64 blkno) { - if (!IS_INDEX(rel) || !IS_BTREE(rel)) + if (!IS_INDEX(rel) || !IS_BTREE(rel) || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a %s index", @@ -858,7 +859,7 @@ bt_metap(PG_FUNCTION_ARGS) relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); rel = relation_openrv(relrv, AccessShareLock); - if (!IS_INDEX(rel) || !IS_BTREE(rel)) + if (!IS_INDEX(rel) || !IS_BTREE(rel) || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a %s index", diff --git a/contrib/pageinspect/gistfuncs.c b/contrib/pageinspect/gistfuncs.c index 23a4b49771d45..b3a8b13cab8db 100644 --- a/contrib/pageinspect/gistfuncs.c +++ b/contrib/pageinspect/gistfuncs.c @@ -15,6 +15,7 @@ #include "access/htup_details.h" #include "access/relation.h" #include "catalog/pg_am_d.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "pageinspect.h" @@ -218,7 +219,7 @@ gist_page_items(PG_FUNCTION_ARGS) /* Open the relation */ indexRel = index_open(indexRelid, AccessShareLock); - if (!IS_GIST(indexRel)) + if (!IS_GIST(indexRel) || doltgres_heap_only_refused(indexRel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a %s index", diff --git a/contrib/pageinspect/hashfuncs.c b/contrib/pageinspect/hashfuncs.c index 7fc97d043ce13..959a2493ffc45 100644 --- a/contrib/pageinspect/hashfuncs.c +++ b/contrib/pageinspect/hashfuncs.c @@ -15,6 +15,7 @@ #include "access/relation.h" #include "catalog/pg_am.h" #include "catalog/pg_type.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "pageinspect.h" @@ -421,7 +422,7 @@ hash_bitmap_info(PG_FUNCTION_ARGS) */ indexRel = relation_open(indexRelid, AccessShareLock); - if (!IS_INDEX(indexRel) || !IS_HASH(indexRel)) + if (!IS_INDEX(indexRel) || !IS_HASH(indexRel) || doltgres_heap_only_refused(indexRel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a %s index", diff --git a/contrib/pageinspect/heapfuncs.c b/contrib/pageinspect/heapfuncs.c index 4f0f3bd53e743..f2a883b8ebb6e 100644 --- a/contrib/pageinspect/heapfuncs.c +++ b/contrib/pageinspect/heapfuncs.c @@ -29,6 +29,7 @@ #include "access/relation.h" #include "catalog/pg_am_d.h" #include "catalog/pg_type.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "mb/pg_wchar.h" #include "miscadmin.h" @@ -325,8 +326,8 @@ tuple_data_split_internal(Oid relid, char *tupdata, /* * Sequences always use heap AM, but they don't show that in the catalogs. */ - if (rel->rd_rel->relkind != RELKIND_SEQUENCE && - rel->rd_rel->relam != HEAP_TABLE_AM_OID) + if ((rel->rd_rel->relkind != RELKIND_SEQUENCE && + rel->rd_rel->relam != HEAP_TABLE_AM_OID) || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only heap AM is supported"))); diff --git a/contrib/pageinspect/rawpage.c b/contrib/pageinspect/rawpage.c index f3996dc39fcde..9cc140f813967 100644 --- a/contrib/pageinspect/rawpage.c +++ b/contrib/pageinspect/rawpage.c @@ -19,6 +19,7 @@ #include "access/relation.h" #include "catalog/namespace.h" #include "catalog/pg_type.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "pageinspect.h" @@ -158,7 +159,8 @@ get_raw_page_internal(text *relname, ForkNumber forknum, BlockNumber blkno) relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); rel = relation_openrv(relrv, AccessShareLock); - if (!RELKIND_HAS_STORAGE(rel->rd_rel->relkind)) + if (!RELKIND_HAS_STORAGE(rel->rd_rel->relkind) || + (forknum == MAIN_FORKNUM && doltgres_heap_only_refused(rel))) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot get raw page from relation \"%s\"", diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c index 50a39d98ff301..3cff53865ef01 100644 --- a/contrib/pg_surgery/heap_surgery.c +++ b/contrib/pg_surgery/heap_surgery.c @@ -18,6 +18,7 @@ #include "access/xloginsert.h" #include "catalog/pg_am_d.h" #include "catalog/pg_control.h" +#include "doltgres/hooks.h" #include "miscadmin.h" #include "storage/bufmgr.h" #include "utils/acl.h" @@ -116,7 +117,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt) RelationGetRelationName(rel)), errdetail_relkind_not_supported(rel->rd_rel->relkind))); - if (rel->rd_rel->relam != HEAP_TABLE_AM_OID) + if (rel->rd_rel->relam != HEAP_TABLE_AM_OID || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only heap AM is supported"))); diff --git a/contrib/pg_visibility/pg_visibility.c b/contrib/pg_visibility/pg_visibility.c index dfab0b64cf50b..f0bf8e0bb4937 100644 --- a/contrib/pg_visibility/pg_visibility.c +++ b/contrib/pg_visibility/pg_visibility.c @@ -16,6 +16,7 @@ #include "access/xloginsert.h" #include "catalog/pg_type.h" #include "catalog/storage_xlog.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "storage/bufmgr.h" @@ -105,6 +106,7 @@ pg_visibility_map(PG_FUNCTION_ARGS) tupdesc = pg_visibility_tupdesc(false, false); mapbits = (int32) visibilitymap_get_status(rel, blkno, &vmbuffer); + mapbits = doltgres_visibility_bits(rel, blkno, mapbits); if (vmbuffer != InvalidBuffer) ReleaseBuffer(vmbuffer); values[0] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0); @@ -146,13 +148,15 @@ pg_visibility(PG_FUNCTION_ARGS) tupdesc = pg_visibility_tupdesc(false, true); mapbits = (int32) visibilitymap_get_status(rel, blkno, &vmbuffer); + mapbits = doltgres_visibility_bits(rel, blkno, mapbits); if (vmbuffer != InvalidBuffer) ReleaseBuffer(vmbuffer); values[0] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0); values[1] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0); /* Here we have to explicitly check rel size ... */ - if (blkno < RelationGetNumberOfBlocks(rel)) + if (blkno < RelationGetNumberOfBlocks(rel) && + !doltgres_visibility_kept(rel)) { buffer = ReadBuffer(rel, blkno); LockBuffer(buffer, BUFFER_LOCK_SHARE); @@ -165,7 +169,10 @@ pg_visibility(PG_FUNCTION_ARGS) else { /* As with the vismap, silently return 0 for pages past EOF */ - values[2] = BoolGetDatum(false); + if (!doltgres_visibility_page(rel, blkno, &values[2])) + { + values[2] = BoolGetDatum(false); + } } relation_close(rel, AccessShareLock); @@ -173,6 +180,21 @@ pg_visibility(PG_FUNCTION_ARGS) PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); } +/* + * doltgres_visibility_page gives pg_visibility's page-level all-visible flag + * of a fork table's block, set for each block within the table since the + * fork has no pages and keeps every block all-visible, answering whether the + * fork stores the table. + */ +bool +doltgres_visibility_page(Relation rel, BlockNumber blkno, Datum *all_visible) +{ + if (!doltgres_visibility_kept(rel)) + return false; + *all_visible = BoolGetDatum(blkno < RelationGetNumberOfBlocks(rel)); + return true; +} + /* * Visibility map information for every block in a relation. */ @@ -503,7 +525,8 @@ collect_visibility_data(Oid relid, bool include_pd) info->count = nblocks; /* Create a stream if reading main fork. */ - if (include_pd) + if (include_pd && + !doltgres_visibility_kept(rel)) { p.current_blocknum = 0; p.last_exclusive = nblocks; @@ -541,7 +564,8 @@ collect_visibility_data(Oid relid, bool include_pd) * caller needs it. Use a buffer access strategy, too, to prevent * cache-trashing. */ - if (include_pd) + if (include_pd && + !doltgres_visibility_kept(rel)) { Buffer buffer; Page page; @@ -557,11 +581,13 @@ collect_visibility_data(Oid relid, bool include_pd) } } - if (include_pd) + if (include_pd && + !doltgres_visibility_kept(rel)) { Assert(read_stream_next_buffer(stream, NULL) == InvalidBuffer); read_stream_end(stream); } + doltgres_visibility_collected(rel, include_pd, info); /* Clean up. */ if (vmbuffer != InvalidBuffer) @@ -571,6 +597,19 @@ collect_visibility_data(Oid relid, bool include_pd) return info; } +/* + * doltgres_visibility_collected sets each block of a fork table all-visible + * and all-frozen in collect_visibility_data's bits, and all-visible on its + * page too when the caller asked for pages, since the fork keeps no + * visibility map and has no pages to read. + */ +void +doltgres_visibility_collected(Relation rel, bool include_pd, vbits *info) +{ + if (doltgres_visibility_kept(rel)) + memset(info->bits, include_pd ? (1 << 0) | (1 << 1) | (1 << 2) : (1 << 0) | (1 << 1), info->count); +} + /* * The "strict" version of GetOldestNonRemovableTransactionId(). The * pg_visibility check can tolerate false positives (don't report some of the diff --git a/contrib/pgstattuple/pgstatapprox.c b/contrib/pgstattuple/pgstatapprox.c index 21e0b50fb4bd4..85de651351c9b 100644 --- a/contrib/pgstattuple/pgstatapprox.c +++ b/contrib/pgstattuple/pgstatapprox.c @@ -18,6 +18,7 @@ #include "access/visibilitymap.h" #include "catalog/pg_am_d.h" #include "commands/vacuum.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "storage/bufmgr.h" @@ -345,7 +346,7 @@ pgstattuple_approx_internal(Oid relid, FunctionCallInfo fcinfo) RelationGetRelationName(rel)), errdetail_relkind_not_supported(rel->rd_rel->relkind))); - if (rel->rd_rel->relam != HEAP_TABLE_AM_OID) + if (rel->rd_rel->relam != HEAP_TABLE_AM_OID || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only heap AM is supported"))); diff --git a/contrib/pgstattuple/pgstatindex.c b/contrib/pgstattuple/pgstatindex.c index 3a3f2637bd956..efb7054267dfc 100644 --- a/contrib/pgstattuple/pgstatindex.c +++ b/contrib/pgstattuple/pgstatindex.c @@ -34,6 +34,7 @@ #include "access/relation.h" #include "catalog/namespace.h" #include "catalog/pg_am.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "storage/bufmgr.h" @@ -222,7 +223,7 @@ pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) ReadStream *stream; BlockNumber startblk; - if (!IS_INDEX(rel) || !IS_BTREE(rel)) + if (!IS_INDEX(rel) || !IS_BTREE(rel) || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("relation \"%s\" is not a btree index", @@ -541,7 +542,7 @@ pgstatginindex_internal(Oid relid, FunctionCallInfo fcinfo) */ rel = relation_open(relid, AccessShareLock); - if (!IS_INDEX(rel) || !IS_GIN(rel)) + if (!IS_INDEX(rel) || !IS_GIN(rel) || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("relation \"%s\" is not a GIN index", @@ -631,7 +632,7 @@ pgstathashindex(PG_FUNCTION_ARGS) */ rel = relation_open(relid, AccessShareLock); - if (!IS_INDEX(rel) || !IS_HASH(rel)) + if (!IS_INDEX(rel) || !IS_HASH(rel) || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("relation \"%s\" is not a hash index", diff --git a/contrib/pgstattuple/pgstattuple.c b/contrib/pgstattuple/pgstattuple.c index 6a7f8cb4a7ca5..eba2c352e5580 100644 --- a/contrib/pgstattuple/pgstattuple.c +++ b/contrib/pgstattuple/pgstattuple.c @@ -32,6 +32,7 @@ #include "access/tableam.h" #include "catalog/namespace.h" #include "catalog/pg_am_d.h" +#include "doltgres/hooks.h" #include "funcapi.h" #include "miscadmin.h" #include "storage/bufmgr.h" @@ -268,7 +269,7 @@ pgstat_relation(Relation rel, FunctionCallInfo fcinfo) errmsg("index \"%s\" is not valid", RelationGetRelationName(rel)))); - switch (rel->rd_rel->relam) + switch (doltgres_heap_only_refused(rel) ? InvalidOid : rel->rd_rel->relam) { case BTREE_AM_OID: return pgstat_index(rel, BTREE_METAPAGE + 1, @@ -328,8 +329,8 @@ pgstat_heap(Relation rel, FunctionCallInfo fcinfo) /* * Sequences always use heap AM, but they don't show that in the catalogs. */ - if (rel->rd_rel->relkind != RELKIND_SEQUENCE && - rel->rd_rel->relam != HEAP_TABLE_AM_OID) + if ((rel->rd_rel->relkind != RELKIND_SEQUENCE && + rel->rd_rel->relam != HEAP_TABLE_AM_OID) || doltgres_heap_only_refused(rel)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only heap AM is supported"))); diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 7eab3cd645050..595f76e2dead3 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -101,6 +101,7 @@ #include "access/visibilitymap.h" #include "access/xloginsert.h" #include "access/xlogutils.h" +#include "doltgres/hooks.h" #include "miscadmin.h" #include "port/pg_bitutils.h" #include "storage/bufmgr.h" @@ -379,6 +380,9 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro /* all_visible must be specified */ Assert(all_visible); + if (doltgres_visibility_counted(rel, all_visible, all_frozen)) + return; + for (mapBlock = 0;; mapBlock++) { Buffer mapBuffer; diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index 0317ead49bf6a..591e451a0eada 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -67,6 +67,7 @@ #include "commands/seclabel.h" #include "commands/tablespace.h" #include "common/file_perm.h" +#include "doltgres/hooks.h" #include "miscadmin.h" #include "postmaster/bgwriter.h" #include "storage/fd.h" @@ -493,6 +494,8 @@ DropTableSpace(DropTableSpaceStmt *stmt) */ deleteSharedDependencyRecordsFor(TableSpaceRelationId, tablespaceoid, 0); + doltgres_tablespace_emptied(tablespaceoid, false); + /* * Acquire TablespaceCreateLock to ensure that no TablespaceCreateDbspace * is running concurrently. @@ -1555,6 +1558,8 @@ tblspc_redo(XLogReaderState *record) /* Close all smgr fds in all backends. */ WaitForProcSignalBarrier(EmitProcSignalBarrier(PROCSIGNAL_BARRIER_SMGRRELEASE)); + doltgres_tablespace_emptied(xlrec->ts_id, true); + /* * If we issued a WAL record for a drop tablespace it implies that * there were no files in it at all when the DROP was done. That means @@ -1574,6 +1579,8 @@ tblspc_redo(XLogReaderState *record) { ResolveRecoveryConflictWithTablespace(xlrec->ts_id); + doltgres_tablespace_emptied(xlrec->ts_id, true); + /* * If we did recovery processing then hopefully the backends who * wrote temp files should have cleaned up and exited by now. So diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 1c575e56ff607..4ef371b7de6a6 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -88,6 +88,7 @@ #include "access/amapi.h" #include "access/htup_details.h" #include "access/tsmapi.h" +#include "doltgres/hooks.h" #include "executor/executor.h" #include "executor/nodeAgg.h" #include "executor/nodeHash.h" @@ -745,6 +746,10 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count, min_IO_cost = 0; } + doltgres_index_scan_io(index->indexoid, indexonly, tuples_fetched, indexSelectivity, loop_count, + spc_random_page_cost, spc_seq_page_cost, cpu_operator_cost, &max_IO_cost, &min_IO_cost, + &rand_heap_pages); + if (partial_path) { /* @@ -1063,6 +1068,9 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, * sqrt(pages_fetched / T); else cost_per_page = spc_random_page_cost; + doltgres_bitmap_page_cost(planner_rt_fetch(baserel->relid, root)->relid, tuples_fetched, pages_fetched, + baserel->pages, tbm_calculate_entries(work_mem * (Size) 1024), spc_random_page_cost, + cpu_operator_cost, &cost_per_page); run_cost += pages_fetched * cost_per_page; diff --git a/src/doltgres/HOOKS.md b/src/doltgres/HOOKS.md index 53d3a58493398..c9002cccca586 100644 --- a/src/doltgres/HOOKS.md +++ b/src/doltgres/HOOKS.md @@ -117,11 +117,12 @@ queue's include-tree check lists. | src/backend/utils/adt/pg_locale_libc.c | `get_collation_actual_version_libc` | call site | libc-named collations report the vendored ICU's collator version (design item 12) | | src/backend/utils/adt/pg_locale_icu.c | `create_pg_locale_icu` | call sites | A libc-named locale's ICU tag comes from the fork, mapped from its stored name, in place of the catalog's locale name, which a libc row leaves null; the two guards skip only the two lines that read it (design item 12) | | src/backend/utils/adt/pg_locale_builtin.c | `create_pg_locale_builtin` | call sites | A libc locale of the C family, but `C` and `POSIX` alone, gets the builtin provider's locale of the name its ctype name stands for (`C.UTF-8` in a UTF-8 database, else `C`), which the fork gives in place of the catalog's locale name, which a libc row leaves null; the two guards skip only the two lines that read it (design item 12) | -| contrib/pageinspect/heapfuncs.c, contrib/pageinspect/rawpage.c, contrib/pageinspect/btreefuncs.c, contrib/pageinspect/brinfuncs.c, contrib/pageinspect/gistfuncs.c, contrib/pageinspect/hashfuncs.c, contrib/pageinspect/ginfuncs.c | each function's access method check | call sites | Refuse fork relations with that check's own SQLSTATE and message (design item 5) | -| contrib/pg_visibility/pg_visibility.c | `check_relation_relkind` | call site | As above (design item 5) | +| contrib/pageinspect/heapfuncs.c, contrib/pageinspect/rawpage.c, contrib/pageinspect/btreefuncs.c, contrib/pageinspect/brinfuncs.c, contrib/pageinspect/gistfuncs.c, contrib/pageinspect/hashfuncs.c | each function's access method check (`rawpage.c`'s is `get_raw_page`'s relkind check; `ginfuncs.c` has none, since its functions take raw pages, which `get_raw_page` refuses) | call sites | Refuse fork relations with that check's own SQLSTATE and message (design item 5) | +| contrib/pg_visibility/pg_visibility.c | `pg_visibility_map`, `pg_visibility`, `collect_visibility_data` | call sites | A fork table keeps no visibility map and has no pages: each block within it reads all-visible and all-frozen, in the map and on its page, and none past it, and its pages are never read (design item 5; bugs row 348) | +| contrib/pg_visibility/pg_visibility.c | `doltgres_visibility_page`, `doltgres_visibility_collected` | helper function | The fork's answers for a fork table that write pg_visibility.c's result values and its private per-block data, `vbits` (design item 5; bugs row 348) | | contrib/pg_surgery/heap_surgery.c | the access method check | call site | As above (design item 5) | | contrib/pgstattuple/pgstattuple.c, contrib/pgstattuple/pgstatapprox.c, contrib/pgstattuple/pgstatindex.c | the access method checks | call sites | As above (design item 5) | -| contrib/amcheck/verify_heapam.c, contrib/amcheck/verify_nbtree.c | the access method checks | call sites | As above (design item 5) | +| contrib/amcheck/verify_heapam.c, contrib/amcheck/verify_common.c | the access method checks (an index's in `verify_common.c`'s `index_checkable`) | call sites | As above (design item 5) | | contrib/pg_prewarm/pg_prewarm.c | `pg_prewarm` | call site | Read a fork relation's trees into the node cache (design item 5) | | src/bin/initdb/initdb.c | `subdirs` | registration entry | Create `pg_versioned` (design item 5) | | src/bin/pg_checksums/pg_checksums.c | `scan_directory` | call site | Skip `pg_versioned` in a tablespace's version directory (design item 5) | @@ -153,7 +154,7 @@ queue's include-tree check lists. | (existing hook) | `RegisterXactCallback`, `XACT_EVENT_ABORT`, `XACT_EVENT_PARALLEL_ABORT`; `RegisterSubXactCallback`, `SUBXACT_EVENT_ABORT_SUB` | existing hook | Take an aborted transaction's or subtransaction's versions back out of the memtable after its abort record (design item 7) | | (existing hook) | `ProcessUtility_hook` | existing hook | After the standard function, settle the tables whose primary key a cascaded drop changed outside ALTER TABLE (design item 5) | | src/backend/optimizer/path/costsize.c | `cost_index` | call site | A fork table's part of an index scan: primary key lookups, none for an index-only scan (design item 16) | -| src/backend/optimizer/path/costsize.c | `cost_bitmap_heap_scan`, `compute_bitmap_pages` | call sites | A fork table's part of a bitmap scan: tid resolutions and primary key lookups (design item 16) | +| src/backend/optimizer/path/costsize.c | `cost_bitmap_heap_scan` | call site | A fork table's cost per page fetched of a bitmap heap scan: a tuple id resolution and a primary key lookup a row, and a tuple id range scan a lossy block (design items 2 and 16) | | src/backend/access/transam/xlogrecovery.c | `PerformWalRecovery` | call site | Refuse a replay that stopped below the WAL position the fork's durable bases and runs prove (design item 8) | | src/backend/utils/adt/float.c | `float4in_internal`, `float8in_internal`, `float4recv`, `float8recv`, `float4send`, `float8send` | call sites | The correctly rounded parser and the quiet positive NaN (design item 15) | | src/port/snprintf.c | `fmtfloat` | call site | Float formatting with a precision through Ryu (design item 15) | diff --git a/src/doltgres/chunks/c/packs.c b/src/doltgres/chunks/c/packs.c index 04dca9bb180b2..91d80d6b830a9 100644 --- a/src/doltgres/chunks/c/packs.c +++ b/src/doltgres/chunks/c/packs.c @@ -434,6 +434,17 @@ chunks_packs_table_runs_insert(StdVec *runs, uint32 space, ChunksPacksTail tail) 298, 27); } +/* chunks_packs_table_runs_remove is BTreeMap::remove of a table's runs tails: a tablespace's taken out when they hold one. */ +void +chunks_packs_table_runs_remove(StdVec *runs, uint32 space) +{ + size_t at; + chunks_TupleU32Tail_t removed; + + if (chunks_packs_table_runs_find(runs, space, &at)) + std_vec_remove(runs, at, &removed, sizeof(removed), _Alignof(chunks_TupleU32Tail_t), RUST_FILE, 0, 0); +} + /* * chunks_packs_Table_whole returns the record that begins a pack table's * file: the generation, the highest number a pack was added at, the bound of diff --git a/src/doltgres/chunks/c/packs/backend.c b/src/doltgres/chunks/c/packs/backend.c index 87692a7af75ac..50e39c60199aa 100644 --- a/src/doltgres/chunks/c/packs/backend.c +++ b/src/doltgres/chunks/c/packs/backend.c @@ -358,7 +358,8 @@ chunks_packs_backend_Closing_body_0(void *frame_) if (!host_shared_read(frame->io, &frame->reading, frame->error)) goto done; frame->reading_live = true; - if (!chunks_packs_backend_Packs_flush(&frame->packs, frame->io, &frame->reading, &failed)) + if (!chunks_packs_backend_Packs_flush(&frame->packs, HOST_IO_IO_TAKES_SHORT(frame->io), &frame->reading, + &failed)) { *frame->error = chunks_packs_shared_raised(&failed); goto done; @@ -567,7 +568,8 @@ chunks_packs_backend_Packs_viewed_body_0(void *frame_) HOST_IO_IO_TAKES_SHORT(frame->io)); frame->view_live = true; view = host_lock_Lending_DerefMut_deref_mut(&frame->view); - frame->ok = chunks_packs_view_View_refresh(view, frame->io, host_shared_Held_record(&frame->self->held), + frame->ok = chunks_packs_view_View_refresh(view, HOST_IO_IO_TAKES_SHORT(frame->io), + host_shared_Held_record(&frame->self->held), frame->error) && frame->read(frame->captures, view, frame->io, frame->error); chunks_packs_backend_Packs_viewed_pad_0(frame); @@ -675,13 +677,16 @@ chunks_packs_backend_Packs_changed(const ChunksPacksBackendPacks *self, HostIoTa /* * chunks_packs_backend_Packs_compact_frame_0_t is compact's state across its - * pad: the view's guard, the list files' flag, and the rewrite under way. + * pad, and remove_space's: the view's guard, the list files' flag, and the + * rewrite under way. */ typedef struct chunks_packs_backend_Packs_compact_frame_0_t { const ChunksPacksBackendPacks *self; HostIoTakesShort io; bool all; + bool removing; + uint32 space; ChunksErrorError *error; bool ok; HostLockLending view; @@ -715,7 +720,7 @@ chunks_packs_backend_Packs_compact_pad_0(void *frame_) } } -/* chunks_packs_backend_Packs_compact_body_0 is compact's body: the list files due a whole write written whole. */ +/* chunks_packs_backend_Packs_compact_body_0 is compact's body, the list files due a whole write written whole, and remove_space's, its tablespace taken out of them. */ static void chunks_packs_backend_Packs_compact_body_0(void *frame_) { @@ -730,7 +735,7 @@ chunks_packs_backend_Packs_compact_body_0(void *frame_) frame->io); frame->view_live = true; view = host_lock_Lending_DerefMut_deref_mut(&frame->view); - if (!frame->all && !chunks_packs_lists_past_slack(view)) + if (!frame->removing && !frame->all && !chunks_packs_lists_past_slack(view)) { frame->ok = true; goto done; @@ -746,7 +751,8 @@ chunks_packs_backend_Packs_compact_body_0(void *frame_) if (!chunks_packs_lists_Rewrite_begin(view, record, &flag_io, &frame->rewrite, frame->error)) goto done; frame->rewrite_live = true; - if (!chunks_packs_lists_Rewrite_compact(&frame->rewrite, frame->all, frame->error)) + if (frame->removing ? !chunks_packs_lists_Rewrite_remove_space(&frame->rewrite, frame->space, frame->error) : + !chunks_packs_lists_Rewrite_compact(&frame->rewrite, frame->all, frame->error)) goto done; frame->rewrite_live = false; frame->ok = chunks_packs_lists_Change_install(&frame->rewrite, frame->error); @@ -770,14 +776,65 @@ chunks_packs_backend_Packs_compact(const ChunksPacksBackendPacks *self, HostIoTa return frame.ok; } -/* chunks_packs_backend_none_deferred is flush's read: whether the view defers no add. */ -static bool -chunks_packs_backend_none_deferred(void *captures, ChunksPacksViewView *view, HostIoIo *io, ChunksErrorError *error) +/* + * chunks_packs_backend_Packs_remove_space takes a tablespace out of the + * repository's list files under their flag once none of its packs is listed, + * refusing while one is (bugs row 99). + */ +bool +chunks_packs_backend_Packs_remove_space(const ChunksPacksBackendPacks *self, HostIoTakesShort io, + const HostSharedReader *reading, uint32 space, ChunksErrorError *error) { - (void) io; - (void) error; - *(bool *) captures = view->deferred.len == 0; - return true; + chunks_packs_backend_Packs_compact_frame_0_t frame = {.self = self, .io = io, .removing = true, .space = space, + .error = error}; + + (void) reading; + std_landing(&frame, chunks_packs_backend_Packs_compact_body_0, chunks_packs_backend_Packs_compact_pad_0); + return frame.ok; +} + +/* + * chunks_packs_backend_Packs_flush_frame_0_t is flush's state across its + * pad: the view's guard, and whether the view defers no add. + */ +typedef struct chunks_packs_backend_Packs_flush_frame_0_t +{ + const ChunksPacksBackendPacks *self; + HostIoTakesShort io; + ChunksErrorError *error; + bool ok; + bool none; + HostLockLending view; + bool view_live; +} chunks_packs_backend_Packs_flush_frame_0_t; + +/* chunks_packs_backend_Packs_flush_pad_0 is flush's pad: the view's guard. */ +static void +chunks_packs_backend_Packs_flush_pad_0(void *frame_) +{ + chunks_packs_backend_Packs_flush_frame_0_t *frame = frame_; + + if (frame->view_live) + { + frame->view_live = false; + host_lock_Lending_drop(&frame->view); + } +} + +/* chunks_packs_backend_Packs_flush_body_0 is flush's body: the view refreshed, then whether it defers no add. */ +static void +chunks_packs_backend_Packs_flush_body_0(void *frame_) +{ + chunks_packs_backend_Packs_flush_frame_0_t *frame = frame_; + ChunksPacksViewView *value; + + frame->view = chunks_packs_backend_Packs_view_lock(frame->self, frame->io); + frame->view_live = true; + value = host_lock_Lending_DerefMut_deref_mut(&frame->view); + frame->ok = chunks_packs_view_View_refresh(value, frame->io, host_shared_Held_record(&frame->self->held), + frame->error); + frame->none = value->deferred.len == 0; + chunks_packs_backend_Packs_flush_pad_0(frame); } /* chunks_packs_backend_flushed is flush's change: every deferred add appended. */ @@ -794,15 +851,16 @@ chunks_packs_backend_flushed(void *captures, ChunksPacksListsChange *change, Chu * its base rests on, so a crash's recovery finds them listed. */ bool -chunks_packs_backend_Packs_flush(const ChunksPacksBackendPacks *self, HostIoIo *io, const HostSharedReader *reading, - ChunksErrorError *error) +chunks_packs_backend_Packs_flush(const ChunksPacksBackendPacks *self, HostIoTakesShort io, + const HostSharedReader *reading, ChunksErrorError *error) { - bool none; + chunks_packs_backend_Packs_flush_frame_0_t frame = {.self = self, .io = io, .error = error}; - if (!chunks_packs_backend_Packs_viewed(self, io, chunks_packs_backend_none_deferred, &none, error)) + std_landing(&frame, chunks_packs_backend_Packs_flush_body_0, chunks_packs_backend_Packs_flush_pad_0); + if (!frame.ok) return false; - return none || chunks_packs_backend_Packs_changed(self, HOST_IO_IO_TAKES_SHORT(io), reading, - chunks_packs_backend_flushed, NULL, error); + return frame.none || chunks_packs_backend_Packs_changed(self, io, reading, chunks_packs_backend_flushed, NULL, + error); } /* @@ -814,7 +872,7 @@ bool chunks_packs_backend_Packs_close(ChunksPacksBackendPacks *self, HostIoIo *io, const HostSharedReader *reading, ChunksErrorError *error) { - bool ok = chunks_packs_backend_Packs_flush(self, io, reading, error); + bool ok = chunks_packs_backend_Packs_flush(self, HOST_IO_IO_TAKES_SHORT(io), reading, error); chunks_packs_backend_Packs_drop(self); return ok; @@ -1062,7 +1120,7 @@ chunks_packs_backend_Packs_copied_body_0(void *frame_) frame->ok = false; if (!chunks_packs_backend_Packs_unreserve(self, frame->io, frame->reading, frame->error) || !chunks_packs_backend_Packs_even_manifest(self, frame->io, frame->error) || - !chunks_packs_backend_Packs_flush(self, frame->io, frame->reading, frame->error) || + !chunks_packs_backend_Packs_flush(self, HOST_IO_IO_TAKES_SHORT(frame->io), frame->reading, frame->error) || !chunks_packs_open_made_dir(frame->io, frame->into, frame->into_len, frame->error)) goto done; frame->dir = host_tablespace_dir((uint32) self->repository, self->own); @@ -2775,7 +2833,7 @@ chunks_packs_backend_Packs_merge_runs_body_0(void *frame_) frame->view = chunks_packs_backend_Packs_view_lock(self, HOST_IO_IO_TAKES_SHORT(frame->io)); frame->view_live = true; value = host_lock_Lending_DerefMut_deref_mut(&frame->view); - if (!chunks_packs_view_View_refresh(value, frame->io, record, frame->error)) + if (!chunks_packs_view_View_refresh(value, HOST_IO_IO_TAKES_SHORT(frame->io), record, frame->error)) goto done; if (!chunks_packs_merge_due(value, &space, &from, &to)) { diff --git a/src/doltgres/chunks/c/packs/fixtures.c b/src/doltgres/chunks/c/packs/fixtures.c index fd23777c4ced5..9e579a345e1bb 100644 --- a/src/doltgres/chunks/c/packs/fixtures.c +++ b/src/doltgres/chunks/c/packs/fixtures.c @@ -613,7 +613,7 @@ chunks_packs_tests_refreshed(const char *file, ChunksPacksViewView *view, HostIo { ChunksErrorError error; - if (!chunks_packs_view_View_refresh(view, io, record, &error)) + if (!chunks_packs_view_View_refresh(view, HOST_IO_IO_TAKES_SHORT(io), record, &error)) chunks_packs_tests_unwrapped(file, &error, line, col); } diff --git a/src/doltgres/chunks/c/packs/lists.c b/src/doltgres/chunks/c/packs/lists.c index 3e35ff0f0947b..4a857d13f2bb3 100644 --- a/src/doltgres/chunks/c/packs/lists.c +++ b/src/doltgres/chunks/c/packs/lists.c @@ -1071,6 +1071,68 @@ chunks_packs_lists_Rewrite_compact(ChunksPacksListsChange *self, bool all, Chunk return chunks_packs_lists_run(&frame, chunks_packs_lists_Rewrite_compact_body_0); } +/* + * chunks_packs_lists_Rewrite_remove_space_body_0 is remove_space's body: the + * deferred adds appended, nothing more when the table holds no runs tail of + * the tablespace, a refusal while the table lists a pack in it, then the + * table written whole without its runs tail, so no record after it names the + * tablespace, and the changer's view's read of its runs file dropped, so its + * next add writes the file whole again. + */ +static void +chunks_packs_lists_Rewrite_remove_space_body_0(void *frame_) +{ + chunks_packs_lists_frame_0_t *frame = frame_; + ChunksPacksListsChange *self = frame->self; + ChunksPacksViewView *view = self->view; + + frame->ok = false; + if (!chunks_packs_lists_flush(frame)) + goto done; + if (chunks_packs_table_runs_get(&view->table.list.runs, frame->space) == NULL) + { + frame->ok = true; + goto done; + } + for (size_t k = 0, k_len = view->table.list.packs.len; k < k_len; k++) + if (STD_VEC_AT(&view->table.list.packs, chunks_TupleU64Listed_t, k).f1.space == frame->space) + { + StdFmtPiece pieces[] = {STD_FMT_LIT("tablespace "), STD_FMT_ARG(0), STD_FMT_LIT(" still holds packs")}; + StdFmtArg args[] = {{&frame->space, std_u32_Display_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + StdString message; + + std_fmt_format_Checked(&message, &arguments); + *frame->error = chunks_error_Error_refused(&message); + goto done; + } + chunks_packs_table_runs_remove(&view->table.list.runs, frame->space); + chunks_packs_lists_path_into(frame, &view->dir, CHUNKS_PACKS_TABLE); + if (!chunks_packs_lists_Rewrite_whole(frame, &CHUNKS_PACKS_VIEW_TABLE_LIST_OPS, &view->table.list, + &view->table.read, &frame->taken_table, sizeof(frame->taken_table))) + goto done; + self->snapshot.table = chunks_packs_lists_tail_of(&CHUNKS_PACKS_VIEW_TABLE_LIST_OPS, &view->table.read, + &view->table.list); + chunks_packs_view_runs_remove(&view->runs, frame->space); + frame->ok = true; +done: + chunks_packs_lists_pad_0(frame); +} + +/* + * chunks_packs_lists_Rewrite_remove_space takes a tablespace out of the + * repository's lists once none of its packs is listed, as DROP TABLESPACE + * needs before it removes the tablespace's directory (bugs row 99), and + * refuses while one is. + */ +bool +chunks_packs_lists_Rewrite_remove_space(ChunksPacksListsChange *self, uint32 space, ChunksErrorError *error) +{ + chunks_packs_lists_frame_0_t frame = {.self = self, .space = space, .error = error}; + + return chunks_packs_lists_run(&frame, chunks_packs_lists_Rewrite_remove_space_body_0); +} + #ifdef DOLTGRES_TEST_CHUNKS /* chunks_packs_lists_tests_TupleU64U64_Debug_fmt is a (u64, u64)'s Debug, in a tuple. */ @@ -1977,7 +2039,7 @@ chunks_packs_lists_tests_reader(void *captures, HostIoIo *io, HostError *error) bool has_two; bool whole; - if (!chunks_packs_view_View_refresh(&view, io, host_shared_Held_record(&held), &failed)) + if (!chunks_packs_view_View_refresh(&view, HOST_IO_IO_TAKES_SHORT(io), host_shared_Held_record(&held), &failed)) { *error = chunks_packs_tests_raised(&failed); ok = false; @@ -2242,4 +2304,114 @@ test_chunks_packs_lists_tests_a_whole_write_that_fails_before_its_renames_leaves simc_TestIo_drop(&test_io); } +/* chunks_packs_lists_tests_first_retired retires pack 1 in a change. */ +static void +chunks_packs_lists_tests_first_retired(ChunksPacksListsChange *change, void *captures) +{ + static const uint64 numbers[] = {1}; + + (void) captures; + chunks_packs_lists_tests_retired(change, numbers, lengthof(numbers), 0, 0); +} + +/* chunks_packs_lists_tests_space_removed takes a tablespace out of a view's repository's list files under their flag, answering whether it could. */ +static bool +chunks_packs_lists_tests_space_removed(HostIoIo *io, HostSharedRecord record, ChunksPacksViewView *view, uint32 space, + ChunksErrorError *error) +{ + HostSharedFlagGuard rewriting; + HostSharedFlagIo flag_io; + ChunksPacksListsChange rewrite; + HostError host; + bool ok; + + if (!host_shared_Record_flag(record, HOST_IO_IO_TAKES_SHORT(io), CHUNKS_PACKS_SHARED_REWRITING_INDEX, &rewriting, + &host)) + chunks_packs_tests_host_unwrapped(RUST_FILE, &host, 0, 0); + flag_io = host_shared_FlagGuard_io(&rewriting); + if (!chunks_packs_lists_Rewrite_begin(view, record, &flag_io, &rewrite, error)) + chunks_packs_tests_unwrapped(RUST_FILE, error, 0, 0); + if (!chunks_packs_lists_Rewrite_remove_space(&rewrite, space, error)) + { + chunks_packs_lists_Change_drop(&rewrite); + ok = false; + } + else + ok = chunks_packs_lists_Change_install(&rewrite, error); + host_shared_FlagGuard_drop(&rewriting); + return ok; +} + +/* + * test_chunks_packs_lists_tests_a_removed_tablespace_reads_a_runs_file_made_again_from_its_start: + * a tablespace's removal is refused while the table lists a pack in it, and + * once its pack is retired it takes the tablespace's runs tail out of the + * table, so a pack the tablespace gets again after its directory went, as a + * tablespace made again under a dropped one's id does, writes a fresh runs + * file every view reads from its start (bugs row 99). + */ +void +test_chunks_packs_lists_tests_a_removed_tablespace_reads_a_runs_file_made_again_from_its_start(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + StdPathBuf dir; + StdPathBuf space; + StdIoError io_error; + HostSharedOwned owned; + HostSharedRecord record; + ChunksPacksFixturesPacked packed; + ChunksPacksFixturesPacked again; + ChunksPacksTableEntry spaced; + ChunksPacksViewView mine; + ChunksPacksViewView other; + ChunksPacksViewView *views[2] = {&mine, &other}; + ChunksErrorError error; + static const uint32 got_cols[] = {0, 0, 0}; + + chunks_packs_fixtures_home(&dir); + space = host_tablespace_dir(CHUNKS_PACKS_FIXTURES_REPOSITORY, 5); + if (!std_fs_create_dir_all(space.bytes.ptr, space.bytes.len, &io_error)) + STD_RESULT_EXPECT_FAILED(0, 0, CHUNKS_PACKS_FIXTURES_UNWRAP, &io_error, std_io_Error_Debug_fmt); + owned = chunks_packs_lists_tests_repository(io, &dir); + record = host_shared_Owned_record(&owned); + packed = chunks_packs_fixtures_pack(io, &space, 1); + chunks_packs_fixtures_view(&mine); + chunks_packs_fixtures_view(&other); + spaced = packed.entry; + spaced.f1 = 5; + chunks_packs_lists_tests_one(io, record, &mine, &spaced, false, 0, 0); + STD_ASSERT_MSG(0, 0, !chunks_packs_lists_tests_space_removed(io, record, &mine, 5, &error) && + error.tag == CHUNKS_ERROR_ERROR_REFUSED, "a tablespace that lists a pack was taken out"); + chunks_error_Error_drop(&error); + chunks_packs_lists_tests_changed(io, record, &mine, chunks_packs_lists_tests_first_retired, NULL); + if (!chunks_packs_lists_tests_space_removed(io, record, &mine, 5, &error)) + chunks_packs_tests_unwrapped(RUST_FILE, &error, 0, 0); + for (size_t k = 0; k < lengthof(views); k++) + { + chunks_packs_tests_refreshed(RUST_FILE, views[k], io, record, 0, 0); + STD_ASSERT(0, 0, chunks_packs_table_runs_get(&chunks_packs_view_View_table(views[k])->runs, 5) == NULL, + "!view.table().runs.contains_key(&5)"); + } + if (!std_fs_remove_dir_all(space.bytes.ptr, space.bytes.len, &io_error) || + !std_fs_create_dir_all(space.bytes.ptr, space.bytes.len, &io_error)) + STD_RESULT_EXPECT_FAILED(0, 0, CHUNKS_PACKS_FIXTURES_UNWRAP, &io_error, std_io_Error_Debug_fmt); + again = chunks_packs_fixtures_pack(io, &space, 2); + spaced = again.entry; + spaced.f1 = 5; + chunks_packs_lists_tests_one(io, record, &mine, &spaced, false, 0, 0); + for (size_t k = 0; k < lengthof(views); k++) + { + chunks_packs_tests_refreshed(RUST_FILE, views[k], io, record, 0, 0); + chunks_packs_tests_listed(RUST_FILE, views[k], &spaced, 1, 0, 0); + chunks_packs_tests_got(RUST_FILE, views[k], io, 5, &again.address, &again.at, 0, got_cols); + } + chunks_packs_view_View_drop(&other); + chunks_packs_view_View_drop(&mine); + host_shared_Owned_drop(&owned); + std_vec_free_Checked(&space.bytes, 1, 1); + std_vec_free_Checked(&dir.bytes, 1, 1); + simc_TestIo_drop(&test_io); +} + #endif /* DOLTGRES_TEST_CHUNKS */ diff --git a/src/doltgres/chunks/c/packs/marking.c b/src/doltgres/chunks/c/packs/marking.c index f97740b076bfd..4bbf42a1fef89 100644 --- a/src/doltgres/chunks/c/packs/marking.c +++ b/src/doltgres/chunks/c/packs/marking.c @@ -977,7 +977,7 @@ chunks_packs_marking_noted_body_0(void *frame_) bool dead; bool recorded; - if (!chunks_packs_view_View_refresh(view, frame->io, record, frame->error)) + if (!chunks_packs_view_View_refresh(view, HOST_IO_IO_TAKES_SHORT(frame->io), record, frame->error)) goto done; if (!host_shared_Record_lock_long_shared(record, frame->io, CHUNKS_PACKS_SHARED_BARRIER_INDEX, &frame->barrier, &host)) diff --git a/src/doltgres/chunks/c/packs/merge.c b/src/doltgres/chunks/c/packs/merge.c index b233d1cd32da0..ca4841033baf4 100644 --- a/src/doltgres/chunks/c/packs/merge.c +++ b/src/doltgres/chunks/c/packs/merge.c @@ -536,7 +536,7 @@ chunks_packs_merge_tests_merged(HostIoIo *io, HostSharedRecord record, ChunksPac ChunksPacksMergeTestsMerged *result, ChunksErrorError *error) { result->has = false; - if (!chunks_packs_view_View_refresh(view, io, record, error)) + if (!chunks_packs_view_View_refresh(view, HOST_IO_IO_TAKES_SHORT(io), record, error)) return false; return chunks_packs_merge_tests_merge_stale(io, record, view, result, error); } diff --git a/src/doltgres/chunks/c/packs/view.c b/src/doltgres/chunks/c/packs/view.c index 240c060dea3de..d9bddd6726859 100644 --- a/src/doltgres/chunks/c/packs/view.c +++ b/src/doltgres/chunks/c/packs/view.c @@ -496,7 +496,7 @@ chunks_packs_view_runs_entry(StdVec *runs, uint32 space) } /* chunks_packs_view_runs_remove drops a tablespace's runs file as the view read it, if it has one. */ -static void +void chunks_packs_view_runs_remove(StdVec *runs, uint32 space) { size_t at; @@ -799,7 +799,7 @@ chunks_packs_view_View_read_lists(ChunksPacksViewView *self, HostIoIoDoors io, c typedef struct chunks_packs_view_View_refresh_frame_0_t { ChunksPacksViewView *self; - HostIoIo *io; + HostIoTakesShort io; HostSharedRecord record; ChunksErrorError *error; bool ok; @@ -863,7 +863,7 @@ chunks_packs_view_View_refresh_body_0(void *frame_) { chunks_packs_view_View_refresh_frame_0_t *frame = frame_; ChunksPacksViewView *self = frame->self; - HostIoIoDoors io = HOST_IO_IO_IO_DOORS(frame->io); + HostIoIoDoors io = HOST_IO_TAKES_SHORT_IO_DOORS(frame->io); uint64 rotations; uint64 generation; bool read; @@ -882,7 +882,7 @@ chunks_packs_view_View_refresh_body_0(void *frame_) } self->has_current = false; rotations = chunks_packs_shared_load(frame->record, CHUNKS_PACKS_SHARED_ROTATIONS_INDEX); - if (!host_shared_Record_lock_shared(frame->record, HOST_IO_IO_TAKES_SHORT(frame->io), + if (!host_shared_Record_lock_shared(frame->record, frame->io, CHUNKS_PACKS_SHARED_STATE_LOCK_INDEX, &frame->state, &host)) { *frame->error = chunks_error_Error_FromError_from(&host); @@ -918,7 +918,7 @@ chunks_packs_view_View_refresh_body_0(void *frame_) #ifdef DOLTGRES_TEST_CHUNKS chunks_packs_view_count_reread(); #endif - if (!host_shared_Record_flag(frame->record, HOST_IO_IO_TAKES_SHORT(frame->io), + if (!host_shared_Record_flag(frame->record, frame->io, CHUNKS_PACKS_SHARED_REWRITING_INDEX, &frame->rewriting, &host)) { *frame->error = chunks_error_Error_FromError_from(&host); @@ -946,7 +946,7 @@ chunks_packs_view_View_refresh_body_0(void *frame_) frame->rewriting_live = false; host_shared_FlagGuard_drop(&frame->rewriting); } - if (!chunks_packs_view_View_open(self, HOST_IO_IO_READ_DOORS(frame->io), frame->snapshot.deferred.ptr, + if (!chunks_packs_view_View_open(self, HOST_IO_IO_DOORS_READ_DOORS(io), frame->snapshot.deferred.ptr, frame->snapshot.deferred.len, frame->error)) goto done; std_vec_free_Checked(&self->deferred, sizeof(ChunksPacksTableEntry), _Alignof(ChunksPacksTableEntry)); @@ -966,7 +966,7 @@ chunks_packs_view_View_refresh_body_0(void *frame_) * a read found a list file missing or not as its tail says. */ bool -chunks_packs_view_View_refresh(ChunksPacksViewView *self, HostIoIo *io, HostSharedRecord record, +chunks_packs_view_View_refresh(ChunksPacksViewView *self, HostIoTakesShort io, HostSharedRecord record, ChunksErrorError *error) { chunks_packs_view_View_refresh_frame_0_t frame = {.self = self, .io = io, .record = record, .error = error}; diff --git a/src/doltgres/chunks/c/trailer.c b/src/doltgres/chunks/c/trailer.c index e1cb0e54eac82..4389a6e38c867 100644 --- a/src/doltgres/chunks/c/trailer.c +++ b/src/doltgres/chunks/c/trailer.c @@ -852,12 +852,13 @@ chunks_trailer_Stamps_holds_closure_3(void *captures, const void *elem) } /* - * chunks_trailer_Stamps_holds reports whether an entry has a stamp of a tuple - * id, by a search of its stamps, which are in tuple id order as a fold writes - * them. + * chunks_trailer_Stamps_stamp_of finds an entry's stamp of a tuple id, by a + * search of its stamps, which are in tuple id order as a fold writes them, + * and answers whether it has one: a run's stamp of a tuple id is its first + * stamp's transaction with that tuple id. */ bool -chunks_trailer_Stamps_holds(const ChunksTrailerStamps *self, size_t i, uint64 tuple) +chunks_trailer_Stamps_stamp_of(const ChunksTrailerStamps *self, size_t i, uint64 tuple, ChunksTrailerStamp *result) { uint64 start; uint64 end; @@ -873,8 +874,12 @@ chunks_trailer_Stamps_holds(const ChunksTrailerStamps *self, size_t i, uint64 tu uint64 at_2; if (!chunks_trailer_Stamps_span(self, i, &start, &end)) - return !self->has_ends && i < self->single.len && - STD_VEC_AT(&self->single, ChunksTrailerStamp, i).tuple == tuple; + { + if (self->has_ends || i >= self->single.len || STD_VEC_AT(&self->single, ChunksTrailerStamp, i).tuple != tuple) + return false; + *result = STD_VEC_AT(&self->single, ChunksTrailerStamp, i); + return true; + } from = std_slice_partition_point(self->runs.ptr, self->runs.len, sizeof(chunks_trailer_Stamps_runs_t), chunks_trailer_Stamps_holds_closure_1, &start, NULL); to = std_slice_partition_point(self->runs.ptr, self->runs.len, sizeof(chunks_trailer_Stamps_runs_t), @@ -898,7 +903,20 @@ chunks_trailer_Stamps_holds(const ChunksTrailerStamps *self, size_t i, uint64 tu else run_start = 0; at_2 = std_saturating_add_u64(run_start, STD_SUB(u64, tuple, first.tuple, 312, 43)); - return at_2 >= std_max_u64(start, run_start) && at_2 < std_min_u64(end, run_end); + if (at_2 < std_max_u64(start, run_start) || at_2 >= std_min_u64(end, run_end)) + return false; + result->txn = first.txn; + result->tuple = tuple; + return true; +} + +/* chunks_trailer_Stamps_holds reports whether an entry has a stamp of a tuple id, as stamp_of finds it. */ +bool +chunks_trailer_Stamps_holds(const ChunksTrailerStamps *self, size_t i, uint64 tuple) +{ + ChunksTrailerStamp stamp; + + return chunks_trailer_Stamps_stamp_of(self, i, tuple, &stamp); } /* @@ -3813,4 +3831,76 @@ test_chunks_trailer_tests_xxh3_digests_are_the_committed_table(void) std_vec_free_Checked(&table, 1, 1); } +/* CHUNKS_TRAILER_TESTS_SPREAD is how many copies the stamp_of case's spread entry holds. */ +#define CHUNKS_TRAILER_TESTS_SPREAD 600 + +/* + * test_chunks_trailer_tests_stamp_of_finds_each_copys_stamp_among_its_runs: + * an entry of copies whose tuple ids are 4 apart, so each is its own run, of + * three transactions, between two other entries, written and read back, + * gives each copy's own stamp by stamp_of, none for a tuple id between, + * past, or of another entry, and the single form gives its one stamp. + */ +void +test_chunks_trailer_tests_stamp_of_finds_each_copys_stamp_among_its_runs(void) +{ + ChunksTrailerStamps stamps = chunks_trailer_Stamps_Default_default(); + ChunksTrailerStamps read; + ChunksTrailerStamps single; + ChunksTrailerStamp before = {1, 1}; + ChunksTrailerStamp after[3] = {{7, 5000}, {7, 5001}, {7, 5002}}; + ChunksTrailerStamp spread[CHUNKS_TRAILER_TESTS_SPREAD]; + ChunksTrailerStamp stamp; + ChunksErrorError error; + StdVec out; + StdVec one; + + for (size_t i = 0; i < CHUNKS_TRAILER_TESTS_SPREAD; i++) + { + spread[i].txn = 100 + i / 200; + spread[i].tuple = 3 + 4 * i; + } + chunks_trailer_Stamps_push(&stamps, &before, 1); + chunks_trailer_Stamps_push(&stamps, spread, CHUNKS_TRAILER_TESTS_SPREAD); + chunks_trailer_Stamps_push(&stamps, after, lengthof(after)); + std_vec_new(&out, 1, 1); + chunks_trailer_Stamps_write(&stamps, &out); + if (!chunks_trailer_Stamps_read(out.ptr, out.len, true, false, &read, &error)) + std_result_unwrap_failed("called `Result::unwrap()` on an `Err` value", + sizeof("called `Result::unwrap()` on an `Err` value") - 1, &error, + chunks_error_Error_Debug_fmt, RUST_FILE, __LINE__, 9); + STD_ASSERT(__LINE__, 9, chunks_trailer_Stamps_entry_len(&read, 1) == CHUNKS_TRAILER_TESTS_SPREAD, + "read.entry_len(1) == SPREAD"); + for (size_t i = 0; i < CHUNKS_TRAILER_TESTS_SPREAD; i++) + STD_ASSERT(__LINE__, 13, + chunks_trailer_Stamps_stamp_of(&read, 1, spread[i].tuple, &stamp) && + chunks_trailer_Stamp_PartialEq_eq(&stamp, &spread[i]), + "read.stamp_of(1, spread[i].tuple) == Some(spread[i])"); + STD_ASSERT(__LINE__, 9, + !chunks_trailer_Stamps_stamp_of(&read, 1, 4, &stamp) && + !chunks_trailer_Stamps_stamp_of(&read, 1, 3 + 4 * CHUNKS_TRAILER_TESTS_SPREAD, &stamp) && + !chunks_trailer_Stamps_stamp_of(&read, 1, 1, &stamp) && + !chunks_trailer_Stamps_stamp_of(&read, 0, 3, &stamp) && + !chunks_trailer_Stamps_stamp_of(&read, 2, 3, &stamp), + "no stamp for a tuple id between, past, or of another entry"); + STD_ASSERT(__LINE__, 9, + chunks_trailer_Stamps_stamp_of(&read, 0, 1, &stamp) && chunks_trailer_Stamp_PartialEq_eq(&stamp, &before) && + chunks_trailer_Stamps_stamp_of(&read, 2, 5001, &stamp) && + chunks_trailer_Stamp_PartialEq_eq(&stamp, &after[1]), + "read.stamp_of(0, 1) == Some(before) && read.stamp_of(2, 5001) == Some(after[1])"); + std_vec_new(&one, sizeof(ChunksTrailerStamp), _Alignof(ChunksTrailerStamp)); + std_vec_push_Checked(&one, &spread[5], sizeof(ChunksTrailerStamp), _Alignof(ChunksTrailerStamp)); + single = chunks_trailer_Stamps_FromVecStamp_from(&one); + STD_ASSERT(__LINE__, 9, + chunks_trailer_Stamps_stamp_of(&single, 0, spread[5].tuple, &stamp) && + chunks_trailer_Stamp_PartialEq_eq(&stamp, &spread[5]) && + !chunks_trailer_Stamps_stamp_of(&single, 0, spread[6].tuple, &stamp) && + !chunks_trailer_Stamps_stamp_of(&single, 1, spread[5].tuple, &stamp), + "the single form's stamp_of gives its one stamp and none else"); + chunks_trailer_Stamps_drop(&single); + chunks_trailer_Stamps_drop(&read); + std_vec_free_Checked(&out, 1, 1); + chunks_trailer_Stamps_drop(&stamps); +} + #endif /* DOLTGRES_TEST_CHUNKS */ diff --git a/src/doltgres/doltgres/c/entries.c b/src/doltgres/doltgres/c/entries.c index 8b24483580cdf..96d2c494c593d 100644 --- a/src/doltgres/doltgres/c/entries.c +++ b/src/doltgres/doltgres/c/entries.c @@ -1574,7 +1574,8 @@ doltgres_entries_rows_closure_0_body(void *frame_) DoltgresScanUnder under = {.state = host_shared_Held_record(frame->held), .snapshot = c->read, .has_serial = c->has_check}; TreesStoreReader store = repo_Repo_reader(frame->repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - DoltgresScanTrees trees = {0}; + uint64 generation = txn_state_generation(under.state); + DoltgresScanTrees trees; uint32 epoch; bool has_chain; TreesStoreTree chain; @@ -1585,14 +1586,7 @@ doltgres_entries_rows_closure_0_body(void *frame_) if (!txn_state_readable(frame->io, frame->held, c->read, &some, &frame->read_base, frame->error)) return; frame->read_base_live = some; - if (some) - { - trees.has_root = frame->read_base.has_root; - trees.root = frame->read_base.root; - trees.has_tuple_ids = frame->read_base.has_tuple_ids; - trees.tuple_ids = frame->read_base.tuple_ids; - trees.folded = frame->read_base.folded; - } + trees = doltgres_scan_Trees_of(some ? &frame->read_base : NULL, generation); doltgres_scan_Epochs_based(some ? &frame->read_base : NULL, &epoch, &has_chain, &chain); if (!doltgres_scan_Epochs_of(frame->io, TREES_STORE_READER_NODE_STORE(&store), epoch, has_chain, chain, &frame->epochs, frame->error)) @@ -1603,9 +1597,13 @@ doltgres_entries_rows_closure_0_body(void *frame_) { DoltgresScanRow row; bool has; + DoltgresScanResolved resolved = {0}; + bool fetched; - if (!doltgres_scan_fetch(frame->io, TREES_STORE_READER_NODE_STORE(&store), &under, &frame->epochs, trees, - &c->layout, c->by.tuple_id, &has, &row, frame->error)) + fetched = doltgres_scan_fetch(frame->io, TREES_STORE_READER_NODE_STORE(&store), &under, &frame->epochs, + &resolved, trees, &c->layout, c->by.tuple_id, &has, &row, frame->error); + doltgres_scan_Resolved_drop(&resolved); + if (!fetched) return; std_vec_new(frame->result, sizeof(DoltgresScanRow), _Alignof(DoltgresScanRow)); if (has) @@ -1960,7 +1958,9 @@ doltgres_entries_raised(bool ok, HostError *failed, PgPgError *error) * subtransaction's start marks where its keys begin, its commit leaves them * to its parent, and its abort, which Postgres records before it calls this, * takes back its versions and its children's by pruning the keys since its - * mark. It marks and ends each subtransaction in the settle's list too. + * mark. It marks and ends each subtransaction in the settle's list too. An + * abort then settles the process's pending spill whose take-out of versions + * from the memtable its error stopped, so the take-out ends there. */ bool doltgres_entries_doltgres_subxact_ended(const PgCaps *caps, int32 event, uint32 mySubid, uint32 parentSubid, @@ -1981,7 +1981,7 @@ doltgres_entries_doltgres_subxact_ended(const PgCaps *caps, int32 event, uint32 { doltgres_layouts_sub_aborted(&io, mySubid); doltgres_settle_sub_ended(&io, mySubid, true); - ok = txn_written_sub_aborted(&io, mySubid, &failed); + ok = txn_written_sub_aborted(&io, mySubid, &failed) && txn_spill_take_settled(&io, &failed); } return doltgres_entries_raised(ok, &failed, error); } @@ -2305,6 +2305,73 @@ doltgres_entries_databases(const PgCatalogCaps *caps, StdVec *result, PgPgError return frame.ok; } +/* + * doltgres_entries_swept empties the fork's packs out of each tablespace whose + * directory of repositories holds the dropped mark, one a standby's redo of + * DROP TABLESPACE couldn't empty, through the collection that empties a + * tablespace, and removes that directory once no repository's is left in it + * (bugs row 99). + */ +static bool +doltgres_entries_swept(HostIoIo *io, StoreCollectSchedule *schedule, const StdVec *databases, + const StoreCollectSettings *settings, PgPgError *error) +{ + StdVec spaces; + HostError failed; + bool ok = true; + + if (!host_tablespaces(HOST_IO_IO_READ_DOORS(io), &spaces, &failed)) + { + *error = pghost_raised(&failed); + return false; + } + for (size_t s = 0, s_len = spaces.len; s < s_len && ok; s++) + { + uint32 space = STD_VEC_AT(&spaces, uint32, s); + StdPathBuf versioned; + StdVec names; + bool has = false; + bool marked = false; + bool left = false; + bool removed; + + versioned = host_versioned_dir(space); + ok = host_list_dir_if_there(HOST_IO_IO_READ_DOORS(io), (const char *) versioned.bytes.ptr, + versioned.bytes.len, &has, &names, &failed); + has = ok && has; + for (size_t k = 0, k_len = has ? names.len : 0; k < k_len; k++) + { + const StdString *name = &STD_VEC_AT(&names, StdString, k); + + marked |= name->len == strlen(DOLTGRES_DROPPED_MARK) && + memcmp(name->ptr, DOLTGRES_DROPPED_MARK, name->len) == 0; + } + for (size_t k = 0, k_len = marked ? names.len : 0; k < k_len && ok; k++) + { + const StdString *name = &STD_VEC_AT(&names, StdString, k); + uint32 repository; + uint64 counts[3]; + StdParseIntError parsed; + + if (!std_str_parse_u32(name->ptr, name->len, &repository, &parsed) || repository == 0) + continue; + left = true; + ok = store_collect_collect_asked(io, schedule, databases->ptr, databases->len, repository, space, + settings, counts, &failed); + } + if (has) + std_vec_drop_Checked(&names, sizeof(StdString), _Alignof(StdString), std_String_drop); + if (ok && marked && !left) + ok = host_remove_tree(HOST_IO_IO_IO_DOORS(io), (const char *) versioned.bytes.ptr, versioned.bytes.len, + &removed, &failed); + std_vec_free_Checked(&versioned.bytes, 1, 1); + if (!ok) + *error = pghost_raised(&failed); + } + std_vec_free_Checked(&spaces, sizeof(uint32), _Alignof(uint32)); + return ok; +} + /* * doltgres_entries_collect_frame_t is collect's state across its pad: the * collector's schedule and the databases of a check, each while live. @@ -2353,6 +2420,7 @@ doltgres_entries_collect_body(void *frame_) HostError failed; uint64 word; uint32 database; + uint32 emptied; if (!pg_calls_latch_sleep(raise, (int64) pg_calls_settings_collector_naptime(), frame->error) || !pg_calls_config_reload_if_pending(raise, catalog, frame->error)) @@ -2362,12 +2430,13 @@ doltgres_entries_collect_body(void *frame_) return; frame->databases_live = true; io = pghost_io(raise); - if (host_collection_asked(&word, &database)) + if (host_collection_asked(&word, &database, &emptied)) { uint64 counts[3] = {PG_UINT64_MAX, PG_UINT64_MAX, PG_UINT64_MAX}; uint64 done[3]; bool ok = store_collect_collect_asked(&io, &frame->schedule, frame->databases.ptr, - frame->databases.len, database, &settings, done, &failed); + frame->databases.len, database, emptied, &settings, done, + &failed); host_collection_answer(word, ok ? done : counts); if (!ok) @@ -2376,6 +2445,8 @@ doltgres_entries_collect_body(void *frame_) return; } } + if (!doltgres_entries_swept(&io, &frame->schedule, &frame->databases, &settings, frame->error)) + return; if (!store_collect_collect_due(&io, &frame->schedule, frame->databases.ptr, frame->databases.len, &settings, &failed)) { @@ -2739,12 +2810,21 @@ doltgres_entries_doltgres_fold_work(const PgCatalogCaps *caps, PgDatum main_arg, * doltgres_entries_doltgres_fold_relation folds a relation's storage now, * inside a fold worker's transaction as its owner, and answers whether it * installed a base; heap is the relation, or an index's table, whose horizon - * bounds the fold. + * bounds the fold. It first pauses at doltgres-fold-relation, with the + * transaction's snapshot taken and the relation locked. */ bool doltgres_entries_doltgres_fold_relation(const PgCatalogCaps *caps, const PgRel *rel, const PgRel *heap, bool *result, PgPgError *error) { + HostIoIo io = pghost_io(pg_CatalogCaps_may_raise(caps)); + HostError failed; + + if (!host_pause(HOST_SHARED_IO_PAUSES(&io), "doltgres-fold-relation", &failed)) + { + *error = pghost_raised(&failed); + return false; + } return doltgres_entries_folded_now(caps, rel, heap, result, error); } @@ -3087,9 +3167,23 @@ doltgres_entries_redo_aborted_body(void *frame_) { doltgres_entries_redo_aborted_frame_t *frame = frame_; doltgres_entries_redo_aborted_captures_t captures = {&frame->rows}; + uint32 aborted = pg_calls_aborted_xid(frame->record); HostIoIo io; HostError failed; + if (aborted != 0) + { + HostXid xid; + + if (!pg_calls_widened_xid(pg_Caps_may_raise(frame->caps), aborted, &xid.f0, frame->error)) + return; + io = pghost_io(pg_Caps_may_raise(frame->caps)); + if (!txn_spill_redo_aborted(&io, xid.f0, pg_calls_record_end(frame->record), &failed)) + { + doltgres_entries_raised(false, &failed, frame->error); + return; + } + } doltgres_entries_filled(DOLTGRES_ENTRIES_RELATION_ROW, pg_calls_aborted_relations, frame->record, &frame->rows); frame->rows_live = true; if (frame->rows.len < DOLTGRES_ENTRIES_RELATION_ROW) @@ -3109,7 +3203,9 @@ doltgres_entries_redo_aborted_body(void *frame_) * deltas the built nodes of every storage a replayed abort drops, at the * start of xact_redo, before any branch's DropRelationFiles, so an aborted * rewrite's Chunk records leave nothing on a standby that never promotes; a - * record of another kind drops none. + * record of another kind drops none. It first marks the replayed aborted + * transaction's free space map spills with the record's end, as its abort + * marked them on its primary. */ bool doltgres_entries_doltgres_redo_aborted(const PgCaps *caps, const PgWalRecord *record, PgPgError *error) @@ -3958,6 +4054,17 @@ doltgres_entries_doltgres_rmgr_redo_closure_11(void *captures, HostIoIo *io, Tre return txn_rewrite_spaced(io, held, c->record, true, c->page.most, true, c->end, error); } +/* doltgres_entries_doltgres_rmgr_redo_closure_12 is a carry's unlist's redo: the carry's versions taken back out. */ +static bool +doltgres_entries_doltgres_rmgr_redo_closure_12(void *captures, HostIoIo *io, TreesStoreNodeStore store, + const HostSharedHeld *held, const HostSharedHeld *toast, + HostError *error) +{ + const doltgres_entries_redone_t *c = captures; + + return txn_spill_redo_unlisted(io, held, c->record, error); +} + /* * doltgres_entries_doltgres_rmgr_redo_frame_t is rmgr_redo's state across its * pad: the record decoded, each while live, a row change's out-of-line @@ -3987,6 +4094,8 @@ typedef struct doltgres_entries_doltgres_rmgr_redo_frame_t bool freed_live; TxnRecordMapCarried carried; bool carried_live; + TxnRecordCarryUnlisted unlisted; + bool unlisted_live; StdArcInner *repo; bool repo_live; HostSharedReader reading; @@ -4023,9 +4132,11 @@ doltgres_entries_doltgres_rmgr_redo_pad(void *frame_) txn_record_TidsFreed_drop(&frame->freed); if (frame->carried_live) txn_record_MapCarried_drop(&frame->carried); + if (frame->unlisted_live) + txn_record_CarryUnlisted_drop(&frame->unlisted); frame->held_live = frame->reading_live = frame->repo_live = frame->entries_live = frame->completion_live = frame->change_live = frame->advanced_live = frame->chunk_live = frame->database_change_live = - frame->freed_live = frame->carried_live = false; + frame->freed_live = frame->carried_live = frame->unlisted_live = false; } /* doltgres_entries_redo_raised raises a redo's host error as Postgres', into the frame's error, answering false. */ @@ -4121,17 +4232,17 @@ doltgres_entries_base_advanced(doltgres_entries_doltgres_rmgr_redo_frame_t *fram if (advanced->horizon != 0 && !pg_calls_resolve_conflict(raise, advanced->horizon, s->tablespace, s->database, s->relation, frame->error)) return false; + end = pg_calls_record_end(frame->record); if (advanced->has_chain) { TxnBaseBase recorded = advanced->base; advanced->base.key.tag = TXN_BASE_KEY_UNKNOWN; if (!txn_fold_redo_chained(&frame->io, repo, &frame->reading, &frame->held, advanced->chain.ptr, - advanced->chain.len, &recorded, &failed)) + advanced->chain.len, &recorded, end, &failed)) return doltgres_entries_redo_raised(frame, &failed); return true; } - end = pg_calls_record_end(frame->record); if (!txn_fold_redo(&frame->io, repo, &frame->reading, &frame->held, advanced, end, &kept, &failed)) return doltgres_entries_redo_raised(frame, &failed); if ((!kept || !shmem_delta_launcher_running()) && @@ -4349,6 +4460,17 @@ doltgres_entries_doltgres_rmgr_redo_body(void *frame_) frame->ok = doltgres_entries_replayed(frame->caps, &frame->carried.storage, NULL, NULL, doltgres_entries_doltgres_rmgr_redo_closure_11, &c, frame->error); return; + case HOST_WAL_KIND_CARRY_UNLISTED: + if (!txn_record_CarryUnlisted_decode(body, body_len, &frame->unlisted, &failed)) + { + doltgres_entries_redo_raised(frame, &failed); + return; + } + frame->unlisted_live = true; + c.record = &frame->unlisted; + frame->ok = doltgres_entries_replayed(frame->caps, &frame->unlisted.storage, NULL, NULL, + doltgres_entries_doltgres_rmgr_redo_closure_12, &c, frame->error); + return; case HOST_WAL_KIND_REPOSITORY_CREATED: frame->ok = doltgres_entries_store_redone(frame, store_databases_redo_created, false, body, body_len); return; @@ -4410,6 +4532,8 @@ typedef struct doltgres_entries_doltgres_rmgr_desc_frame_t bool freed_live; TxnRecordMapCarried carried; bool carried_live; + TxnRecordCarryUnlisted unlisted; + bool unlisted_live; StdString text; bool text_live; } doltgres_entries_doltgres_rmgr_desc_frame_t; @@ -4434,8 +4558,10 @@ doltgres_entries_doltgres_rmgr_desc_pad(void *frame_) txn_record_TidsFreed_drop(&frame->freed); if (frame->carried_live) txn_record_MapCarried_drop(&frame->carried); + if (frame->unlisted_live) + txn_record_CarryUnlisted_drop(&frame->unlisted); frame->text_live = frame->change_live = frame->advanced_live = frame->chunk_live = false; - frame->database_change_live = frame->freed_live = frame->carried_live = false; + frame->database_change_live = frame->freed_live = frame->carried_live = frame->unlisted_live = false; } /* doltgres_entries_described formats a description into the frame's text. */ @@ -4678,6 +4804,30 @@ doltgres_entries_rmgr_desc_carried(doltgres_entries_doltgres_rmgr_desc_frame_t * return true; } +/* doltgres_entries_rmgr_desc_unlisted describes a carry's unlist: the storage and how many of the carry's records' ends it names. */ +static bool +doltgres_entries_rmgr_desc_unlisted(doltgres_entries_doltgres_rmgr_desc_frame_t *frame, const uint8 *body, + size_t body_len, HostError *failed) +{ + StdFmtPiece pieces[] = {STD_FMT_LIT("rel "), STD_FMT_ARG(0), STD_FMT_LIT("/"), STD_FMT_ARG(1), STD_FMT_LIT("/"), + STD_FMT_ARG(2), STD_FMT_LIT("; "), STD_FMT_ARG(3), STD_FMT_LIT(" ends")}; + const HostStorage *s; + size_t ends; + + if (!txn_record_CarryUnlisted_decode(body, body_len, &frame->unlisted, failed)) + return false; + frame->unlisted_live = true; + s = &frame->unlisted.storage; + ends = frame->unlisted.ends.len; + { + StdFmtArg args[] = {{&s->tablespace, std_u32_Display_fmt}, {&s->database, std_u32_Display_fmt}, + {&s->relation, std_u32_Display_fmt}, {&ends, std_usize_Display_fmt}}; + + doltgres_entries_described(frame, pieces, lengthof(pieces), args); + } + return true; +} + /* doltgres_entries_doltgres_rmgr_desc_body describes the record by its kind, then appends the description. */ static void doltgres_entries_doltgres_rmgr_desc_body(void *frame_) @@ -4723,6 +4873,9 @@ doltgres_entries_doltgres_rmgr_desc_body(void *frame_) case HOST_WAL_KIND_MAP_CARRIED: ok = doltgres_entries_rmgr_desc_carried(frame, body, body_len, &failed); break; + case HOST_WAL_KIND_CARRY_UNLISTED: + ok = doltgres_entries_rmgr_desc_unlisted(frame, body, body_len, &failed); + break; case HOST_WAL_KIND_REPOSITORY_CREATED: ok = store_databases_describe_created(body, body_len, &frame->text, &failed); frame->text_live = ok; @@ -4867,44 +5020,57 @@ typedef struct doltgres_entries_committing_t /* * doltgres_entries_committing_closure_0 is committing's lent call over a * value map: each written value's entry with the transaction's versions - * against without, recorded for the commit to add. + * against without, recorded for the commit to add, read again from the + * newest base when an install moved the base before the entries were read + * (bugs rows 240 and 316). It pauses at doltgres-commit-valued after each + * read of the base. */ static bool doltgres_entries_committing_closure_0(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, HostError *error) { const doltgres_entries_committing_t *c = captures; - TreesStoreReader store; - TxnBaseSizes adds = {0}; - TxnBaseSizes takes = {0}; - TxnBaseBase base; - TreesStoreTree root = {0}; - bool has_root = false; - bool some; + HostSharedRecord record = host_shared_Held_record(held); + TreesStoreReader store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - if (!txn_state_base_record(io, held, &some, &base, error)) - return false; - if (some) - { - has_root = base.has_root; - root = base.root; - txn_base_Base_drop(&base); - } - store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - for (size_t i = 0; i < c->keys->len; i++) + for (;;) { - const StdVec *key = &STD_VEC_AT(c->keys, StdVec, i); - TxnBaseSizes added; - TxnBaseSizes taken; + uint64 generation = txn_state_generation(record); + TxnBaseSizes adds = {0}; + TxnBaseSizes takes = {0}; + TxnBaseBase base; + TreesStoreTree root = {0}; + bool has_root = false; + bool some; - if (!txn_fold_committed_value_moves(io, TREES_STORE_READER_NODE_STORE(&store), held, has_root, root, c->own, - key->ptr, key->len, &added, &taken, error)) + if (!txn_state_base_record(io, held, &some, &base, error)) + return false; + if (some) + { + has_root = base.has_root; + root = base.root; + txn_base_Base_drop(&base); + } + if (!host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-commit-valued", error)) return false; - adds = txn_base_Sizes_summed(adds, added); - takes = txn_base_Sizes_summed(takes, taken); + for (size_t i = 0; i < c->keys->len; i++) + { + const StdVec *key = &STD_VEC_AT(c->keys, StdVec, i); + TxnBaseSizes added; + TxnBaseSizes taken; + + if (!txn_fold_committed_value_moves(io, TREES_STORE_READER_NODE_STORE(&store), held, has_root, root, + c->own, key->ptr, key->len, &added, &taken, error)) + return false; + adds = txn_base_Sizes_summed(adds, added); + takes = txn_base_Sizes_summed(takes, taken); + } + if (!txn_state_moved_past(record, generation)) + { + txn_state_committing(held, adds, takes); + return true; + } } - txn_state_committing(held, adds, takes); - return true; } /* doltgres_entries_committing_closure_1 is committing's based: the table and index its storage's base names. */ @@ -4939,48 +5105,63 @@ doltgres_entries_committing_closure_2(void *captures, HostIoIo *io, const RepoRe /* * doltgres_entries_committing_closure_3 is committing's lent call over a * table's or an index's rows: each written key's fold over the base with the - * transaction's versions against without, recorded for the commit to add. + * transaction's versions against without, recorded for the commit to add, + * folded again over the newest base when an install moved the base before + * the keys' versions were read (bugs rows 240 and 316). It pauses at + * doltgres-commit-based after each read of the base. */ static bool doltgres_entries_committing_closure_3(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, HostError *error) { const doltgres_entries_committing_t *c = captures; - TreesStoreReader store; - TxnBaseSizes adds = {0}; - TxnBaseSizes takes = {0}; - TxnBaseBase base; - TreesStoreTree root; - TreesStoreTree tuple_ids; - bool has_root; - bool has_tuple_ids; - bool some; + HostSharedRecord record = host_shared_Held_record(held); + TreesStoreReader store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - if (!txn_state_base_record(io, held, &some, &base, error)) - return false; - if (!some) - return true; - has_root = base.has_root; - root = base.root; - has_tuple_ids = base.has_tuple_ids; - tuple_ids = base.tuple_ids; - txn_base_Base_drop(&base); - store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - for (size_t i = 0; i < c->keys->len; i++) + for (;;) { - const StdVec *key = &STD_VEC_AT(c->keys, StdVec, i); - TxnBaseSizes added; - TxnBaseSizes taken; - - if (!txn_fold_committed_moves(io, TREES_STORE_READER_NODE_STORE(&store), held, has_root, root, has_tuple_ids, - tuple_ids, c->order, c->keyless, c->index, c->own, key->ptr, key->len, &added, - &taken, error)) + uint64 generation = txn_state_generation(record); + TxnBaseSizes adds = {0}; + TxnBaseSizes takes = {0}; + TxnBaseBase base; + TreesStoreTree root; + TreesStoreTree tuple_ids; + bool has_root; + bool has_tuple_ids; + HostLsn folded; + bool some; + + if (!txn_state_base_record(io, held, &some, &base, error)) return false; - adds = txn_base_Sizes_summed(adds, added); - takes = txn_base_Sizes_summed(takes, taken); + if (!some) + return true; + has_root = base.has_root; + root = base.root; + has_tuple_ids = base.has_tuple_ids; + tuple_ids = base.tuple_ids; + folded = base.folded; + txn_base_Base_drop(&base); + if (!host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-commit-based", error)) + return false; + for (size_t i = 0; i < c->keys->len; i++) + { + const StdVec *key = &STD_VEC_AT(c->keys, StdVec, i); + TxnBaseSizes added; + TxnBaseSizes taken; + + if (!txn_fold_committed_moves(io, TREES_STORE_READER_NODE_STORE(&store), held, has_root, root, + has_tuple_ids, tuple_ids, c->order, c->keyless, c->index, folded, c->own, + key->ptr, key->len, &added, &taken, error)) + return false; + adds = txn_base_Sizes_summed(adds, added); + takes = txn_base_Sizes_summed(takes, taken); + } + if (!txn_state_moved_past(record, generation)) + { + txn_state_committing(held, adds, takes); + return true; + } } - txn_state_committing(held, adds, takes); - return true; } /* doltgres_entries_TupleStorageOfVecVecU8_drop is a written tree's drop glue: its keys. */ @@ -5234,7 +5415,9 @@ doltgres_entries_committing(const PgCatalogCaps *caps, PgPgError *error) * which Postgres records before it calls this, it takes back the * transaction's versions by pruning those keys (2g's take-back). Each end * clears the moves of the totals. It ends the settle's list at a commit, a - * prepare, or an abort (the rewrite text's section 6). + * prepare, or an abort (the rewrite text's section 6). An abort first + * settles the process's pending spill whose take-out of versions from the + * memtable its error stopped, so the take-out ends there. */ bool doltgres_entries_doltgres_xact_ended(const PgCatalogCaps *caps, int32 event, void *arg, PgPgError *error) @@ -5274,7 +5457,7 @@ doltgres_entries_doltgres_xact_ended(const PgCatalogCaps *caps, int32 event, voi } else if (abort) { - if (!txn_written_aborted(&io, &failed)) + if (!txn_spill_take_settled(&io, &failed) || !txn_written_aborted(&io, pg_calls_last_record_end(), &failed)) return doltgres_entries_raised(false, &failed, error); doltgres_settle_ended(&io, true); } @@ -5448,25 +5631,31 @@ doltgres_entries_handle(const PgScan *scan) #define TXN_STATE_RESULT_ERROR error #include "doltgres/txn/state_template.h" -/* doltgres_entries_doltgres_scan_begin_captures_t is scan_begin's lent call's captures: the relation and the read. */ +/* + * doltgres_entries_doltgres_scan_begin_captures_t is scan_begin's lent + * call's captures: the relation, the read, and the base generation its base + * was read at. + */ typedef struct doltgres_entries_doltgres_scan_begin_captures_t { const PgRel *relation; const HostSnapshot *read; + uint64 generation; } doltgres_entries_doltgres_scan_begin_captures_t; /* * doltgres_entries_doltgres_scan_begin_closure_0 is scan_begin's lent call: * the storage's block count and the base its read sees, or its newest when - * it has no snapshot. + * it has no snapshot, with the generation read before it. */ static bool doltgres_entries_doltgres_scan_begin_closure_0(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, uint64 *result_0, bool *some, TxnBaseBase *result_1, HostError *error) { - const doltgres_entries_doltgres_scan_begin_captures_t *c = captures; + doltgres_entries_doltgres_scan_begin_captures_t *c = captures; + c->generation = txn_state_generation(host_shared_Held_record(held)); if (c->read != NULL ? !txn_state_readable(io, held, c->read, some, result_1, error) : !txn_state_newest(io, held, some, result_1, error)) return false; @@ -5543,10 +5732,48 @@ doltgres_entries_doltgres_scan_begin_pad(void *frame_) frame->columns_live = frame->key_recorded_live = frame->reading_live = frame->repo_live = false; } +/* txn::state::held at u64, which entries_tail.c defines and scan_begin and analyze_next_tuple take. */ +#define TXN_STATE_TS U64 +#define TXN_STATE_RETURNS bool +#define TXN_STATE_RESULT_PARAMS , uint64 *result, HostError *error +#define TXN_STATE_DECLARE +#include "doltgres/txn/state_template.h" + +/* + * doltgres_entries_doltgres_scan_begin_closure_1 is a parallel scan's lent + * call: the base generation its worker reads at, one before the state's when + * the base the leader shared is no longer the storage's newest, so the + * worker's first read moves to the newest (rebased). + */ +static bool +doltgres_entries_doltgres_scan_begin_closure_1(void *captures, HostIoIo *io, const RepoRepo *repo, + const HostSharedHeld *held, uint64 *result, HostError *error) +{ + const DoltgresScanShared *shared = captures; + uint64 generation = txn_state_generation(host_shared_Held_record(held)); + TxnBaseBase newest; + bool some; + bool same; + + if (!txn_state_newest(io, held, &some, &newest, error)) + return false; + same = !some || + (newest.has_root == shared->has_tree && + (!newest.has_root || trees_store_Tree_PartialEq_eq(&newest.root, &shared->tree)) && + newest.has_tuple_ids == shared->has_tuple_ids && + (!newest.has_tuple_ids || trees_store_Tree_PartialEq_eq(&newest.tuple_ids, &shared->tuple_ids)) && + newest.read == shared->folded); + if (some) + txn_base_Base_drop(&newest); + *result = same ? generation : generation - 1; + return true; +} + /* * doltgres_entries_doltgres_scan_begin_parallel makes a parallel scan's * record over the base the leader shared: its trees, its split keys, its - * epochs, and its count of entries and units. + * epochs, its count of entries and units, and the base generation its + * worker reads at. */ static bool doltgres_entries_doltgres_scan_begin_parallel(doltgres_entries_doltgres_scan_begin_frame_t *frame, @@ -5561,9 +5788,20 @@ doltgres_entries_doltgres_scan_begin_parallel(doltgres_entries_doltgres_scan_beg bool has_chain; TreesStoreReader store; HostError failed; + uint64 generation; frame->shared.has_tree = doltgres_entries_root(&parts.table, &frame->shared.tree); frame->shared.has_tuple_ids = doltgres_entries_root(&parts.tuples, &frame->shared.tuple_ids); + frame->shared.folded = parts.folded; + if (!doltgres_entries_making(frame->caps, &frame->io, frame->relation, &frame->found, frame->error)) + return false; + if (!txn_state_held_U64(&frame->io, repo, &frame->reading, frame->found, + doltgres_entries_doltgres_scan_begin_closure_1, &frame->shared, NULL, &generation, + &failed)) + { + *frame->error = pghost_raised(&failed); + return false; + } if (!doltgres_scan_split_keys(splits, splits_len, &frame->splits, &failed)) { *frame->error = pghost_raised(&failed); @@ -5572,7 +5810,6 @@ doltgres_entries_doltgres_scan_begin_parallel(doltgres_entries_doltgres_scan_beg frame->splits_live = true; frame->shared.fresh_end = parts.fresh; frame->shared.entries = parts.entries; - frame->shared.folded = parts.folded; frame->shared.units = units; store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, &frame->reading); has_chain = doltgres_entries_root(&parts.chain, &chain); @@ -5584,8 +5821,8 @@ doltgres_entries_doltgres_scan_begin_parallel(doltgres_entries_doltgres_scan_beg } frame->shared.splits = frame->splits; frame->splits_live = frame->keyed_live = frame->order_live = false; - if (!doltgres_scan_begin_parallel(&frame->io, &frame->shared, &frame->keyed, pg_calls_rel_id(frame->relation), - &frame->order, owned, &failed)) + if (!doltgres_scan_begin_parallel(&frame->io, &frame->shared, generation, &frame->keyed, + pg_calls_rel_id(frame->relation), &frame->order, owned, &failed)) { *frame->error = pghost_raised(&failed); return false; @@ -5630,7 +5867,7 @@ doltgres_entries_doltgres_scan_begin_serial(doltgres_entries_doltgres_scan_begin store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, &frame->reading); frame->keyed_live = frame->order_live = false; if (!doltgres_scan_scan_begin(&frame->io, TREES_STORE_READER_NODE_STORE(&store), blocks, - has_base ? &frame->read_base : NULL, &frame->keyed, + has_base ? &frame->read_base : NULL, c.generation, &frame->keyed, pg_calls_rel_id(frame->relation), &frame->order, owned, &failed)) { *frame->error = pghost_raised(&failed); @@ -6413,13 +6650,6 @@ doltgres_entries_sampled(HostIoIo *io, const DoltgresScanRow *row, DoltgresEntri return true; } -/* txn::state::held at u64, which entries_tail.c defines and analyze_next_tuple takes. */ -#define TXN_STATE_TS U64 -#define TXN_STATE_RETURNS bool -#define TXN_STATE_RESULT_PARAMS , uint64 *result, HostError *error -#define TXN_STATE_DECLARE -#include "doltgres/txn/state_template.h" - /* doltgres_entries_spent_t is analyze_next_tuple's lent call's captures: the reader and the block's first tid. */ typedef struct doltgres_entries_spent_t { diff --git a/src/doltgres/doltgres/c/entries/searches.c b/src/doltgres/doltgres/c/entries/searches.c index 649d366aa2a2d..7470f2f41392e 100644 --- a/src/doltgres/doltgres/c/entries/searches.c +++ b/src/doltgres/doltgres/c/entries/searches.c @@ -2661,12 +2661,26 @@ doltgres_entries_searches_read_closure_5(void *captures, HostIoIo *io, DoltgresE return true; } -/* doltgres_entries_searches_read_closure_3 is read's lent call: the base the scan's snapshot reads, as splits' is. */ +/* doltgres_entries_searches_read_captures_t is read's lent call's captures: the snapshot and the generation it reads. */ +typedef struct doltgres_entries_searches_read_captures_t +{ + const HostSnapshot *read; + uint64 generation; +} doltgres_entries_searches_read_captures_t; + +/* + * doltgres_entries_searches_read_closure_3 is read's lent call: the base the + * scan's snapshot reads, as splits' is, with the base generation read + * before it. + */ static bool doltgres_entries_searches_read_closure_3(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, bool *some, TxnBaseBase *result, HostError *error) { - return doltgres_entries_searches_Entries_splits_closure_0(captures, io, repo, held, some, result, error); + doltgres_entries_searches_read_captures_t *c = captures; + + c->generation = txn_state_generation(host_shared_Held_record(held)); + return doltgres_entries_searches_Entries_splits_closure_0((void *) c->read, io, repo, held, some, result, error); } /* doltgres_entries_searches_unsnapshotted returns the error of an index scan asked to read with no snapshot. */ @@ -2828,6 +2842,7 @@ doltgres_entries_searches_read_body_0(void *frame_) TreesStoreReader store; TreesKeyOrder order; bool ok; + doltgres_entries_searches_read_captures_t c = {&frame->read}; if (!host_owned_TableDoltgresEntriesSearchesSearch_withVecU8(&DOLTGRES_ENTRIES_SEARCHES_SEARCHES, &frame->io, frame->owned, @@ -2843,8 +2858,8 @@ doltgres_entries_searches_read_body_0(void *frame_) doltgres_entries_index_making(&frame->io, index, &found, frame->error))) return; if (!txn_state_held_OptionBase(&frame->io, STD_ARC_VALUE(frame->repo, _Alignof(RepoRepo)), &frame->reading, - found, doltgres_entries_searches_read_closure_3, &frame->read, NULL, &some, - &base, &failed)) + found, doltgres_entries_searches_read_closure_3, &c, NULL, &some, &base, + &failed)) { *frame->error = pghost_raised(&failed); return; @@ -2854,7 +2869,8 @@ doltgres_entries_searches_read_body_0(void *frame_) trees_KeyOrder_Clone_clone(&frame->order, &order); frame->keyed_live = false; ok = doltgres_scan_scan_begin(&frame->io, TREES_STORE_READER_NODE_STORE(&store), 0, some ? &base : NULL, - &frame->keyed, pg_calls_index_table(index), &order, &reader, &failed); + c.generation, &frame->keyed, pg_calls_index_table(index), &order, &reader, + &failed); if (some) txn_base_Base_drop(&base); if (!ok || diff --git a/src/doltgres/doltgres/c/entries/testing.c b/src/doltgres/doltgres/c/entries/testing.c index 57ba3e3ce8f09..c1734a4c83640 100644 --- a/src/doltgres/doltgres/c/entries/testing.c +++ b/src/doltgres/doltgres/c/entries/testing.c @@ -32,10 +32,12 @@ #include "doltgres/chunks/error.h" #include "doltgres/chunks/file.h" #include "doltgres/chunks/pack.h" +#include "doltgres/chunks/packs.h" #include "doltgres/chunks/packs/roots.h" #include "doltgres/chunks/packs/shared.h" #include "doltgres/chunks/trailer.h" #include "doltgres/doltgres/entries.h" +#include "doltgres/doltgres/entries_tail.h" #include "doltgres/doltgres/entries/testing.h" #include "doltgres/doltgres/index.h" #include "doltgres/doltgres/kinds.h" @@ -93,6 +95,7 @@ #include "doltgres/txn/memtable/table.h" #include "doltgres/txn/memtable/version.h" #include "doltgres/txn/rewrite.h" +#include "doltgres/txn/runs.h" #include "doltgres/txn/spill.h" #include "doltgres/txn/state.h" #include "doltgres/txn/status.h" @@ -1346,19 +1349,23 @@ doltgres_entries_testing_doltgres_test_wal(const PgCaps *caps, int64 value, int6 /* * doltgres_entries_testing_doltgres_test_wal_unknown inserts and flushes a - * record of a kind no crate owns, of len zero bytes, whose redo stops - * recovery. + * record of a kind no crate owns, the lowest value no kind of the registry + * has, of len zero bytes, whose redo stops recovery. */ bool doltgres_entries_testing_doltgres_test_wal_unknown(const PgCaps *caps, int32 len, PgPgError *error) { const PgMayRaise *raise = pg_Caps_may_raise(caps); + uint8 unknown = 0; + HostWalKind kind; StdVec body; uint64 lsn; bool ok; + while (host_WalKind_of(unknown, &kind)) + unknown++; std_vec_u8_from_elem(&body, 0, len > 0 ? (size_t) len : 0); - ok = pg_calls_wal_insert(raise, 14, (const uint8 *) body.ptr, body.len, &lsn, error) && + ok = pg_calls_wal_insert(raise, unknown, (const uint8 *) body.ptr, body.len, &lsn, error) && pg_calls_wal_flush(raise, lsn, error); std_vec_free_Checked(&body, 1, 1); return ok; @@ -7112,7 +7119,7 @@ doltgres_entries_testing_doltgres_test_collect(const PgCatalogCaps *caps, PgDatu HostError failed; if (!host_session_database(&database, &tablespace, &failed) || - !host_collection_ask(&io, database, counts, &failed)) + !host_collection_ask(&io, database, CHUNKS_PACKS_DEFAULT_SPACE, counts, &failed)) { *error = pghost_raised(&failed); return false; @@ -7316,7 +7323,7 @@ doltgres_entries_testing_identity_body_0(void *frame_) uint64 id; StdString shown; - frame->io = pghost_io(raise); + frame->io = pghost_io_catalog(frame->caps); if (!doltgres_entries_repository(&frame->io, &frame->repo, frame->error)) return; frame->repo_live = true; @@ -7402,6 +7409,169 @@ doltgres_entries_testing_doltgres_test_identity(const PgCatalogCaps *caps, int32 return frame.ok; } +/* + * doltgres_entries_testing_identity_versions_frame_0_t is + * doltgres_test_identity_versions's state across its pad: the working + * state's delta, a read's hold, the key, and the versions read, each while + * live. + */ +typedef struct doltgres_entries_testing_identity_versions_frame_0_t +{ + const PgCaps *caps; + int32 kind; + uint32 number; + HostIoIo io; + HostSharedHeld deltas; + bool deltas_live; + HostDeltaHold hold; + bool hold_live; + StdVec key; + bool key_live; + StdVec versions; + bool versions_live; + StdVec spilled; + bool spilled_live; + PgDatum *result; + PgPgError *error; + bool ok; +} doltgres_entries_testing_identity_versions_frame_0_t; + +/* doltgres_entries_testing_identity_versions_body_0 counts the key's versions in the delta and its identity spills. */ +static void +doltgres_entries_testing_identity_versions_body_0(void *frame_) +{ + doltgres_entries_testing_identity_versions_frame_0_t *frame = frame_; + const PgMayRaise *raise = pg_Caps_may_raise(frame->caps); + HostError failed; + uint32 database = pg_calls_my_database_id(); + bool some; + uint64 count = 0; + StdString shown; + + frame->io = pghost_io(raise); + if (database == 0) + { + *frame->error = doltgres_entries_testing_pg_fork("a backend has no database"); + return; + } + if (!txn_memtable_branch_find(HOST_SHARED_IO_WITHIN(&frame->io), database, &some, &frame->deltas, &failed)) + { + *frame->error = pghost_raised(&failed); + return; + } + if (some) + { + HostSharedRecord deltas = host_shared_Held_record(&frame->deltas); + + frame->deltas_live = true; + if (!host_delta_hold(&frame->io, &frame->hold, &failed)) + { + *frame->error = pghost_raised(&failed); + return; + } + frame->hold_live = true; + txn_identity_by_oid(frame->kind == 1 ? TXN_IDENTITY_KIND_SCHEMA : TXN_IDENTITY_KIND_RELATION, frame->number, + &frame->key); + frame->key_live = true; + if (!txn_memtable_branch_identity_versions(&frame->io, &frame->hold, deltas, frame->key.ptr, frame->key.len, + &frame->versions, &failed)) + { + *frame->error = pghost_raised(&failed); + return; + } + frame->versions_live = true; + if (!txn_spill_versions(&TXN_SPILL_IDENTITY_HOME_OPS, &frame->io, &frame->hold, deltas, + TXN_RUNS_KIND_SPILLED_IDENTITY, frame->key.ptr, frame->key.len, NULL, &frame->spilled, + &failed)) + { + *frame->error = pghost_raised(&failed); + return; + } + frame->spilled_live = true; + count = (uint64) (frame->versions.len + frame->spilled.len); + } + { + StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&count, std_u64_Display_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + std_fmt_format_Checked(&shown, &arguments); + } + frame->ok = doltgres_entries_testing_text_datum(raise, &shown, "a count's text holds no NUL", __LINE__, 47, + frame->result, frame->error); +} + +/* doltgres_entries_testing_identity_versions_pad_0 drops the versions, the key, the hold, and the delta. */ +static void +doltgres_entries_testing_identity_versions_pad_0(void *frame_) +{ + doltgres_entries_testing_identity_versions_frame_0_t *frame = frame_; + + if (frame->spilled_live) + txn_memtable_version_versions_drop(&frame->spilled, 0); + if (frame->versions_live) + txn_memtable_version_versions_drop(&frame->versions, 0); + if (frame->key_live) + std_vec_free_Checked(&frame->key, 1, 1); + if (frame->hold_live) + host_delta_Hold_drop(&frame->hold); + if (frame->deltas_live) + host_shared_Held_drop(&frame->deltas); + frame->spilled_live = frame->versions_live = frame->key_live = frame->hold_live = frame->deltas_live = false; +} + +/* + * doltgres_entries_testing_doltgres_test_identity_versions returns, as text, + * how many versions the database's working state's delta and its identity + * spills hold of the identity tree's key of a schema's OID for kind 1 or a + * relation's for kind 2 (bugs row 291). + */ +bool +doltgres_entries_testing_doltgres_test_identity_versions(const PgCaps *caps, int32 kind, uint32 number, + PgDatum *result, PgPgError *error) +{ + doltgres_entries_testing_identity_versions_frame_0_t frame = {.caps = caps, .kind = kind, .number = number, + .result = result, .error = error}; + + std_landing(&frame, doltgres_entries_testing_identity_versions_body_0, + doltgres_entries_testing_identity_versions_pad_0); + doltgres_entries_testing_identity_versions_pad_0(&frame); + return frame.ok; +} + +/* + * doltgres_entries_testing_doltgres_test_save runs item 6's save outside a + * checkpoint for the backend's database's repository, as a states' pool's + * make does, and returns whether it took an identity version out and + * whether it turned a state it gathered clean, as two numbers. + */ +bool +doltgres_entries_testing_doltgres_test_save(const PgCaps *caps, PgDatum *result, PgPgError *error) +{ + const PgMayRaise *raise = pg_Caps_may_raise(caps); + HostIoIo io = pghost_io(raise); + StdArcInner *repo; + uint32 own; + bool took; + bool cleaned; + HostError failed; + uint64 numbers[2]; + + if (!doltgres_entries_repository(&io, &repo, error)) + return false; + own = repo_Repo_own(STD_ARC_VALUE(repo, _Alignof(RepoRepo))); + std_arc_drop(repo, sizeof(RepoRepo), _Alignof(RepoRepo), doltgres_entries_testing_Repo_drop); + if (!txn_runs_save(&io, pg_calls_my_database_id(), own, &took, &cleaned, &failed)) + { + *error = pghost_raised(&failed); + return false; + } + numbers[0] = took; + numbers[1] = cleaned; + return doltgres_entries_testing_numbers_datum(raise, numbers, lengthof(numbers), "two numbers hold no NUL", + __LINE__, 58, result, error); +} + /* state.rs' held at Option, which entries.c defines. */ #define TXN_STATE_TS OptionBase #define TXN_STATE_RETURNS bool @@ -7812,6 +7982,71 @@ doltgres_entries_testing_doltgres_test_base_sizes(const PgCatalogCaps *caps, con return doltgres_entries_testing_held_run(&frame, doltgres_entries_testing_base_sizes_body_0); } +static void doltgres_entries_testing_hex(StdBytes bytes, StdString *result); + +/* doltgres_entries_testing_base_points_body_0 reads the newest base's root, fold point, and read point as their text. */ +static void +doltgres_entries_testing_base_points_body_0(void *frame_) +{ + doltgres_entries_testing_held_frame_0_t *frame = frame_; + const PgMayRaise *raise = pg_CatalogCaps_may_raise(frame->caps); + HostError failed; + bool some; + StdString shown; + + if (!doltgres_entries_testing_opened(frame)) + return; + if (!txn_state_held_OptionBase(&frame->io, doltgres_entries_testing_held_repo(frame), &frame->reading, + frame->found, doltgres_entries_testing_doltgres_test_base_sizes_closure_0, NULL, NULL, &some, + &frame->base, &failed)) + { + *frame->error = pghost_raised(&failed); + return; + } + frame->base_live = some; + if (some) + { + StdString root; + StdFmtPiece pieces[] = {STD_FMT_ARG(0), STD_FMT_LIT(" "), STD_FMT_ARG(1), STD_FMT_LIT(" "), STD_FMT_ARG(2)}; + StdFmtArg args[] = {{&root, std_String_Display_fmt}, {&frame->base.folded, std_u64_Display_fmt}, + {&frame->base.read, std_u64_Display_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + if (frame->base.has_root) + { + StdBytes address = {frame->base.root.address.f0, sizeof(frame->base.root.address.f0)}; + + doltgres_entries_testing_hex(address, &root); + } + else + std_vec_new(&root, 1, 1); + std_fmt_format_Checked(&shown, &arguments); + std_String_drop(&root); + frame->base_live = false; + txn_base_Base_drop(&frame->base); + } + else + std_string_from_Checked(&shown, "none", 4); + frame->ok = doltgres_entries_testing_text_datum(raise, &shown, "a base's points' text holds no NUL", 2899, 47, + frame->datum, frame->error); +} + +/* + * doltgres_entries_testing_doltgres_test_base_points returns a table's + * storage's newest base's root as its address in hex, with its fold point and + * its read point, or "none" for a storage with no base, so a standby's base + * is compared with its primary's. + */ +bool +doltgres_entries_testing_doltgres_test_base_points(const PgCatalogCaps *caps, const PgRel *target, PgDatum *result, + PgPgError *error) +{ + doltgres_entries_testing_held_frame_0_t frame = {.caps = caps, .target = target, .datum = result, + .error = error}; + + return doltgres_entries_testing_held_run(&frame, doltgres_entries_testing_base_points_body_0); +} + /* doltgres_entries_testing_doltgres_test_held_closure_0 is doltgres_test_held's lent call: whether a version holds the tuple id. */ static bool doltgres_entries_testing_doltgres_test_held_closure_0(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, @@ -7959,6 +8194,33 @@ doltgres_entries_testing_doltgres_test_totals(const PgCatalogCaps *caps, const P return doltgres_entries_testing_held_run(&frame, doltgres_entries_testing_totals_body_0); } +/* + * doltgres_entries_testing_doltgres_test_scan_shape returns a fork table's + * shape as its scans' costs read it: its tree's leaves and levels above + * them, then its tuple id tree's. + */ +bool +doltgres_entries_testing_doltgres_test_scan_shape(const PgCatalogCaps *caps, const PgRel *target, PgDatum *result, + PgPgError *error) +{ + DoltgresEntriesScanned shape; + uint64 heights[2]; + StdString shown; + StdFmtPiece pieces[] = {STD_FMT_ARG(0), STD_FMT_LIT(" "), STD_FMT_ARG(1), STD_FMT_LIT(" "), STD_FMT_ARG(2), + STD_FMT_LIT(" "), STD_FMT_ARG(3)}; + StdFmtArg args[] = {{&shape.rows.leaves, std_u64_Display_fmt}, {&heights[0], std_u64_Display_fmt}, + {&shape.tuple_ids.leaves, std_u64_Display_fmt}, {&heights[1], std_u64_Display_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + if (!doltgres_entries_scanned(caps, 0, pg_calls_rel_id(target), &shape, error)) + return false; + heights[0] = shape.rows.height; + heights[1] = shape.tuple_ids.height; + std_fmt_format_Checked(&shown, &arguments); + return doltgres_entries_testing_text_datum(pg_CatalogCaps_may_raise(caps), &shown, + "a scan shape's text holds no NUL", __LINE__, 0, result, error); +} + /* doltgres_entries_testing_op_name returns the name doltgres_test_changes gives a version's op. */ static StdStr doltgres_entries_testing_op_name(TxnMemtableVersionOpRefTag op) diff --git a/src/doltgres/doltgres/c/entries_index.c b/src/doltgres/doltgres/c/entries_index.c index 9fc91ec7d1e17..ab36ce93d165e 100644 --- a/src/doltgres/doltgres/c/entries_index.c +++ b/src/doltgres/doltgres/c/entries_index.c @@ -1421,21 +1421,28 @@ typedef struct DoltgresEntriesOthers StdBytes keyed; } DoltgresEntriesOthers; -/* doltgres_entries_others_closure_1 is others' read of the index's base the snapshot sees. */ -static bool -doltgres_entries_others_closure_1(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, - bool *some, TxnBaseBase *result, HostError *error) -{ - return txn_state_readable(io, held, captures, some, result, error); -} - -/* doltgres_entries_others_captures_t is others' next's captures: the snapshot and the reader. */ +/* + * doltgres_entries_others_captures_t is others' lent calls' captures: the + * snapshot, the base generation its base was read at, and the reader. + */ typedef struct doltgres_entries_others_captures_t { const HostSnapshot *seen; + uint64 generation; HostOwnedHandle reader; } doltgres_entries_others_captures_t; +/* doltgres_entries_others_closure_1 is others' read of the index's base the snapshot sees, with the generation before it. */ +static bool +doltgres_entries_others_closure_1(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, + bool *some, TxnBaseBase *result, HostError *error) +{ + doltgres_entries_others_captures_t *c = captures; + + c->generation = txn_state_generation(host_shared_Held_record(held)); + return txn_state_readable(io, held, c->seen, some, result, error); +} + /* doltgres_entries_others_closure_2 is others' next: the reader's next entry under the snapshot. */ static bool doltgres_entries_others_closure_2(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, @@ -1531,7 +1538,7 @@ doltgres_entries_others_entries(doltgres_entries_others_frame_t *frame) if (!doltgres_entries_index_making(&frame->io, which->index, &found, frame->error)) return false; if (!txn_state_held_OptionBase(&frame->io, which->repo, which->reading, found, doltgres_entries_others_closure_1, - (void *) frame->seen, NULL, &some, &frame->base, &failed)) + &c, NULL, &some, &frame->base, &failed)) { *frame->error = pghost_raised(&failed); return false; @@ -1543,7 +1550,8 @@ doltgres_entries_others_entries(doltgres_entries_others_frame_t *frame) std_slice_to_vec_Checked(&keyed, which->keyed.ptr, which->keyed.len, 1, 1); frame->order_live = false; if (!doltgres_scan_scan_begin(&frame->io, TREES_STORE_READER_NODE_STORE(&store), 0, some ? &frame->base : NULL, - &keyed, pg_calls_index_table(which->index), &frame->order, &c.reader, &failed) || + c.generation, &keyed, pg_calls_index_table(which->index), &frame->order, &c.reader, + &failed) || !doltgres_scan_seek(&frame->io, &stores, c.reader, which->prefix.ptr, which->prefix.len, DOLTGRES_SCAN_DIRECTION_FORWARD, &failed)) { @@ -1964,11 +1972,10 @@ doltgres_entries_doltgres_index_insert_killed_op(void *value, void *captures) } } -/* doltgres_entries_doltgres_index_insert_captures_t is index_insert's key read's captures: the reader, the table's base, and the row's tuple id. */ +/* doltgres_entries_doltgres_index_insert_captures_t is index_insert's key read's captures: the reader and the row's tuple id. */ typedef struct doltgres_entries_doltgres_index_insert_captures_t { const HostSharedReader *reading; - const TxnBaseBase *base; uint64 tuple; } doltgres_entries_doltgres_index_insert_captures_t; @@ -1981,9 +1988,8 @@ doltgres_entries_doltgres_index_insert_closure_1(void *captures, HostIoIo *io, c const doltgres_entries_doltgres_index_insert_captures_t *c = captures; TreesStoreReader store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - return doltgres_scan_keyed(io, TREES_STORE_READER_NODE_STORE(&store), host_shared_Held_record(held), - c->base->has_tuple_ids, c->base->tuple_ids, c->tuple, some, result_0, - result_1, error); + return doltgres_scan_keyed(io, TREES_STORE_READER_NODE_STORE(&store), held, c->tuple, some, result_0, result_1, + error); } /* doltgres_entries_Described_drop_glue is an Arc's value's drop glue. */ @@ -2101,8 +2107,7 @@ doltgres_entries_doltgres_index_insert_body(void *frame_) uint64 tuple = pg_calls_tid_value(frame->heap_tid); doltgres_entries_doltgres_index_insert_killed_captures_t killed = {.relation = pg_calls_rel_id(frame->heap), .tuple = tuple}; - doltgres_entries_doltgres_index_insert_captures_t c = {.reading = &frame->opened.reading, .base = &frame->base, - .tuple = tuple}; + doltgres_entries_doltgres_index_insert_captures_t c = {.reading = &frame->opened.reading, .tuple = tuple}; const RepoRepo *repo; const DoltgresIndexDescribed *described; TxnMemtableVersionWriter writer; diff --git a/src/doltgres/doltgres/c/entries_tail.c b/src/doltgres/doltgres/c/entries_tail.c index 022250ae32998..c05f6dec8bb7c 100644 --- a/src/doltgres/doltgres/c/entries_tail.c +++ b/src/doltgres/doltgres/c/entries_tail.c @@ -25,6 +25,7 @@ #include "access/nbtree.h" #include "access/tableam.h" +#include "access/visibilitymapdefs.h" #include "access/xact.h" #include "catalog/pg_class_d.h" #include "commands/progress.h" @@ -338,8 +339,6 @@ doltgres_entries_chained_closure_0_body(void *frame_) TreesStoreReader store = repo_Repo_reader(frame->repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); TxnBaseBase newest; bool some; - bool has_tuple_ids = false; - TreesStoreTree tuple_ids = {0}; HostLsn folded = 0; bool has_key; bool keyless; @@ -350,13 +349,11 @@ doltgres_entries_chained_closure_0_body(void *frame_) return; if (some) { - has_tuple_ids = newest.has_tuple_ids; - tuple_ids = newest.tuple_ids; - folded = newest.folded; + folded = newest.read; txn_base_Base_drop(&newest); } - if (!doltgres_scan_keyed(frame->io, TREES_STORE_READER_NODE_STORE(&store), host_shared_Held_record(frame->held), - has_tuple_ids, tuple_ids, c->tuple, &has_key, &frame->key, &keyless, frame->error)) + if (!doltgres_scan_keyed(frame->io, TREES_STORE_READER_NODE_STORE(&store), frame->held, c->tuple, &has_key, + &frame->key, &keyless, frame->error)) return; *frame->some = false; if (!has_key) @@ -1214,7 +1211,7 @@ doltgres_entries_doltgres_relation_nontransactional_truncate(const PgCatalogCaps static void doltgres_entries_Carry_drop(void *carry) { - txn_record_RowChange_drop(&((DoltgresRewriteCarry *) carry)->f0); + txn_record_RowChange_drop(&((TxnRewriteCarry *) carry)->f0); } /* doltgres_entries_Copied_drop is Copied's drop glue, an element of copied's answer. */ @@ -1380,7 +1377,7 @@ doltgres_entries_doltgres_relation_copy_for_cluster_closure_5_body(void *frame_) { doltgres_entries_doltgres_relation_copy_for_cluster_closure_5_frame_t *frame = frame_; doltgres_entries_doltgres_relation_copy_for_cluster_captures_t *c = frame->c; - const DoltgresRewriteCarry *carried = c->carried->ptr; + const TxnRewriteCarry *carried = c->carried->ptr; TxnInsertPage page = doltgres_entries_page(); bool some; bool first = true; @@ -1433,10 +1430,9 @@ doltgres_entries_doltgres_relation_copy_for_cluster_closure_5_body(void *frame_) return; } store = repo_Repo_reader(frame->repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - for (size_t i = 0; i < c->carried->len; i++) - if (!txn_rewrite_carried(frame->io, TREES_STORE_READER_NODE_STORE(&store), frame->held, &carried[i].f0, - carried[i].f1, carried[i].f2, c->logged, &page, frame->error)) - return; + if (!txn_rewrite_carried(frame->io, TREES_STORE_READER_NODE_STORE(&store), frame->held, NULL, 0, carried, + c->carried->len, c->logged, &page, pg_calls_max_heap_tuples_per_page(), frame->error)) + return; *frame->result = true; frame->ok = true; } @@ -1524,7 +1520,7 @@ doltgres_entries_doltgres_relation_copy_for_cluster_pad(void *frame_) std_vec_drop_Checked(&frame->copied, sizeof(TxnRewriteCopied), _Alignof(TxnRewriteCopied), doltgres_entries_Copied_drop); if (frame->carried_live) - std_vec_drop_Checked(&frame->carried, sizeof(DoltgresRewriteCarry), _Alignof(DoltgresRewriteCarry), + std_vec_drop_Checked(&frame->carried, sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry), doltgres_entries_Carry_drop); if (frame->toast_live) host_shared_Held_drop(&frame->values); @@ -1693,7 +1689,7 @@ doltgres_entries_doltgres_relation_copy_for_cluster_body(void *frame_) return; for (size_t i = 0; i < frame->carried.len; i++) { - const TxnMemtableVersionWriter *writer = &STD_VEC_AT(&frame->carried, DoltgresRewriteCarry, i).f0.writer; + const TxnMemtableVersionWriter *writer = &STD_VEC_AT(&frame->carried, TxnRewriteCarry, i).f0.writer; uint64 xid = writer->has_top ? writer->top : writer->xid; if (xid < TXN_STATUS_FIRST_NORMAL || (has_oldest && xid >= oldest)) @@ -1830,7 +1826,7 @@ typedef struct doltgres_entries_moved_into_captures_t bool logged; const TxnRecordMapCarried *spaces; size_t spaces_len; - const DoltgresRewriteCarry *carried; + const TxnRewriteCarry *carried; size_t carried_len; } doltgres_entries_moved_into_captures_t; @@ -1869,14 +1865,38 @@ doltgres_entries_moved_into_closure_0_body(void *frame_) doltgres_entries_moved_into_closure_0_frame_t *frame = frame_; doltgres_entries_moved_into_captures_t *c = frame->c; HostLsn point = c->newest->folded; + HostLsn read; bool copied; TreesStoreReader store; + StdVec writers; + bool ok; std_vec_new(&frame->copies.copies, sizeof(txn_TupleLocatorLocator_t), _Alignof(txn_TupleLocatorLocator_t)); frame->copies_live = true; *c->newest_live = false; - if (!txn_fold_copied(frame->io, frame->repo, c->reading, frame->held, c->newest, point, c->has_nodes, c->nodes, - &frame->copies, &copied, frame->error)) + std_vec_new(&writers, sizeof(uint64), _Alignof(uint64)); + for (size_t i = 0; i < c->spaces_len; i++) + for (size_t v = 0; v < c->spaces[i].versions.len; v++) + { + const TxnMemtableVersionWriter *writer = &STD_VEC_AT(&c->spaces[i].versions, TxnRecordCarriedTake, + v).writer; + uint64 top = writer->has_top ? writer->top : writer->xid; + + std_vec_push_Checked(&writers, &top, sizeof(top), _Alignof(uint64)); + } + for (size_t i = 0; i < c->carried_len; i++) + { + const TxnMemtableVersionWriter *writer = &c->carried[i].f0.writer; + uint64 top = writer->has_top ? writer->top : writer->xid; + + std_vec_push_Checked(&writers, &top, sizeof(top), _Alignof(uint64)); + } + ok = txn_rewrite_moved_point(frame->io, c->newest->read, writers.ptr, writers.len, &read, frame->error); + std_vec_free_Checked(&writers, sizeof(uint64), _Alignof(uint64)); + if (!ok) + return; + if (!txn_fold_copied(frame->io, frame->repo, c->reading, frame->held, c->newest, point, read, c->has_nodes, + c->nodes, &frame->copies, &copied, frame->error)) return; if (!copied) { @@ -1884,15 +1904,11 @@ doltgres_entries_moved_into_closure_0_body(void *frame_) frame->ok = true; return; } - for (size_t i = 0; i < c->spaces_len; i++) - if (!txn_rewrite_spaced(frame->io, frame->held, &c->spaces[i], c->logged, pg_calls_max_heap_tuples_per_page(), - false, 0, frame->error)) - return; store = repo_Repo_reader(frame->repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - for (size_t i = 0; i < c->carried_len; i++) - if (!txn_rewrite_carried(frame->io, TREES_STORE_READER_NODE_STORE(&store), frame->held, &c->carried[i].f0, - c->carried[i].f1, c->carried[i].f2, c->logged, NULL, frame->error)) - return; + if (!txn_rewrite_carried(frame->io, TREES_STORE_READER_NODE_STORE(&store), frame->held, c->spaces, c->spaces_len, + c->carried, c->carried_len, c->logged, NULL, pg_calls_max_heap_tuples_per_page(), + frame->error)) + return; *frame->result = true; frame->ok = true; } @@ -1997,7 +2013,7 @@ doltgres_entries_moved_into_body(void *frame_) static bool doltgres_entries_moved_into(const PgMayRaise *raise, HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const PgRel *rel, HostStorage moved, TxnBaseBase *newest, - const TxnRecordMapCarried *spaces, size_t spaces_len, const DoltgresRewriteCarry *carried, + const TxnRecordMapCarried *spaces, size_t spaces_len, const TxnRewriteCarry *carried, size_t carried_len, PgPgError *error) { doltgres_entries_moved_into_frame_t frame = {.newest = *newest, .newest_live = true, @@ -2066,7 +2082,7 @@ doltgres_entries_doltgres_relation_copy_data_closure_0_pad(void *frame_) doltgres_entries_doltgres_relation_copy_data_closure_0_frame_t *frame = frame_; if (frame->carried_live) - std_vec_drop_Checked(frame->carried, sizeof(DoltgresRewriteCarry), _Alignof(DoltgresRewriteCarry), + std_vec_drop_Checked(frame->carried, sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry), doltgres_entries_Carry_drop); if (frame->base_live) txn_base_Base_drop(frame->base); @@ -2102,7 +2118,7 @@ doltgres_entries_doltgres_relation_copy_data_closure_0_body(void *frame_) if (!doltgres_entries_moved_base(frame->io, frame->held, frame->base, frame->error)) return; frame->base_live = true; - if (!doltgres_rewrite_moving(frame->io, frame->held, frame->base->folded, c->moved, c->table, false, 0, + if (!doltgres_rewrite_moving(frame->io, frame->held, frame->base->read, c->moved, c->table, false, 0, frame->carried, frame->error)) return; frame->carried_live = true; @@ -2175,7 +2191,7 @@ doltgres_entries_doltgres_relation_copy_data_pad(void *frame_) std_vec_drop_Checked(&frame->spaces, sizeof(TxnRecordMapCarried), _Alignof(TxnRecordMapCarried), doltgres_entries_MapCarried_drop); if (frame->carried_live) - std_vec_drop_Checked(&frame->carried, sizeof(DoltgresRewriteCarry), _Alignof(DoltgresRewriteCarry), + std_vec_drop_Checked(&frame->carried, sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry), doltgres_entries_Carry_drop); if (frame->base_live) txn_base_Base_drop(&frame->base); @@ -2208,7 +2224,7 @@ doltgres_entries_doltgres_relation_copy_data_named(doltgres_entries_doltgres_rel &frame->opened.reading); for (size_t i = 0; i < frame->carried.len; i++) { - TxnRecordRowChange *change = &STD_VEC_AT(&frame->carried, DoltgresRewriteCarry, i).f0; + TxnRecordRowChange *change = &STD_VEC_AT(&frame->carried, TxnRewriteCarry, i).f0; const TxnMemtablePut *put; const uint32 *ids = NULL; size_t ids_len = 0; @@ -2326,7 +2342,7 @@ doltgres_entries_doltgres_relation_copy_data_body(void *frame_) { const TxnMemtablePut *put; - valued = txn_record_RowChange_put(&STD_VEC_AT(&frame->carried, DoltgresRewriteCarry, i).f0, &put) && + valued = txn_record_RowChange_put(&STD_VEC_AT(&frame->carried, TxnRewriteCarry, i).f0, &put) && put->values.len > 0; } if (valued && !doltgres_entries_doltgres_relation_copy_data_named(frame)) @@ -2429,7 +2445,7 @@ doltgres_entries_doltgres_index_copy_data_closure_0_body(void *frame_) if (!doltgres_entries_moved_base(frame->io, frame->held, frame->base, frame->error)) return; frame->base_live = true; - if (!doltgres_rewrite_moving(frame->io, frame->held, frame->base->folded, c->moved, c->table, true, c->index, + if (!doltgres_rewrite_moving(frame->io, frame->held, frame->base->read, c->moved, c->table, true, c->index, frame->carried, frame->error)) return; frame->base_live = false; @@ -2480,7 +2496,7 @@ doltgres_entries_doltgres_index_copy_data_pad(void *frame_) doltgres_entries_doltgres_index_copy_data_frame_t *frame = frame_; if (frame->carried_live) - std_vec_drop_Checked(&frame->carried, sizeof(DoltgresRewriteCarry), _Alignof(DoltgresRewriteCarry), + std_vec_drop_Checked(&frame->carried, sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry), doltgres_entries_Carry_drop); frame->carried_live = false; doltgres_entries_opened_drop(&frame->opened); @@ -2888,7 +2904,8 @@ doltgres_entries_doltgres_relation_size_body(void *frame_) /* * doltgres_entries_doltgres_relation_size is the fork's relation_size: the * main fork's block count times the block size, from the storage's tree's - * totals and its tuple id counter (PLAN item 5's sizes for costs), and 0 for + * totals and its tuple id counter (PLAN item 5's sizes for costs), the free + * space map fork's own pages, which Postgres keeps (bugs row 87), and 0 for * every other fork, which the fork's relations don't have. */ bool @@ -2898,6 +2915,15 @@ doltgres_entries_doltgres_relation_size(const PgCatalogCaps *caps, const PgRel * doltgres_entries_doltgres_relation_size_frame_t frame = {.caps = caps, .rel = rel, .result = result, .error = error}; + if (forkNumber == FSM_FORKNUM) + { + uint32 blocks; + + if (!pg_calls_fork_blocks(pg_CatalogCaps_may_raise(caps), rel, forkNumber, &blocks, error)) + return false; + *result = (uint64) blocks * BLCKSZ; + return true; + } if (forkNumber != MAIN_FORKNUM) { *result = 0; @@ -3038,7 +3064,8 @@ doltgres_entries_size_added(int64 size, uint64 added) * nothing, in pg_relation_size for the fork it names and in the sums over a * table's relations for every fork: a table's block count and an index's, * times BLCKSZ, and nothing for a relation the fork doesn't store or another - * fork, which has no file. + * fork, which has no file but the free space map's, whose pages Postgres + * summed. */ bool doltgres_entries_doltgres_relation_size_added(const PgCatalogCaps *caps, const PgRel *rel, const PgText *forkName, @@ -3065,13 +3092,393 @@ doltgres_entries_doltgres_relation_size_added(const PgCatalogCaps *caps, const P if (!doltgres_relation_stored(caps, rel, &stored, error)) return false; - if (stored && !doltgres_entries_doltgres_relation_size(caps, rel, fork_number, &added, error)) + if (stored && fork_number == MAIN_FORKNUM && + !doltgres_entries_doltgres_relation_size(caps, rel, fork_number, &added, error)) return false; } *result = doltgres_entries_size_added(size, added); return true; } +/* + * doltgres_entries_doltgres_visibility_counted counts every block of a table + * the fork stores all-visible and all-frozen, in visibilitymap_count, since + * the fork keeps no visibility map and every block is both (design items 5 + * and 16): its block count, as RelationGetNumberOfBlocks gives it, the + * all-frozen count only where its caller asks for one; it answers nothing + * for a relation the fork doesn't store. + */ +bool +doltgres_entries_doltgres_visibility_counted(const PgCatalogCaps *caps, const PgRel *rel, uint32 *all_visible, + uint32 *all_frozen, bool *result, PgPgError *error) +{ + uint64 bytes; + + if (!doltgres_relation_stored(caps, rel, result, error)) + return false; + if (!*result) + return true; + if (!doltgres_entries_doltgres_relation_size(caps, rel, MAIN_FORKNUM, &bytes, error)) + return false; + *all_visible = (uint32) ((bytes + (BLCKSZ - 1)) / BLCKSZ); + if (all_frozen != NULL) + *all_frozen = *all_visible; + return true; +} + +/* + * doltgres_entries_doltgres_heap_only_refused answers whether the fork + * stores a relation, a table by access's rule and an index by whether its + * entries are the fork's, for the heap-only contrib modules' checks (PLAN + * item 5), which refuse it as they refuse a relation of another access + * method. + */ +bool +doltgres_entries_doltgres_heap_only_refused(const PgCatalogCaps *caps, const PgRel *rel, bool *result, + PgPgError *error) +{ + if (pg_calls_rel_kind(rel) == RELKIND_INDEX) + return doltgres_index_ported(pg_CatalogCaps_may_raise(caps), pg_CatalogCaps_catalog(caps), rel, result, + error); + return doltgres_relation_stored(caps, rel, result, error); +} + +/* + * doltgres_entries_doltgres_visibility_kept answers whether the fork stores + * a relation, for pg_visibility, which then reads none of its pages and + * answers each of its blocks all-visible and all-frozen, since the fork keeps + * no visibility map (design item 5): none of the fork's reads consults a + * block's bits, and none of its versions ever needs a freeze. Its + * pg_check_visible and pg_check_frozen find nothing because it has no map + * fork to name a block to check. + */ +bool +doltgres_entries_doltgres_visibility_kept(const PgCatalogCaps *caps, const PgRel *rel, bool *result, PgPgError *error) +{ + return doltgres_relation_stored(caps, rel, result, error); +} + +/* + * doltgres_entries_doltgres_visibility_bits gives pg_visibility a block's + * visibility map bits: for a table the fork stores, all-visible and + * all-frozen for a block within its block count, as RelationGetNumberOfBlocks + * gives it, and neither past it; for any other relation, the map's own bits. + */ +bool +doltgres_entries_doltgres_visibility_bits(const PgCatalogCaps *caps, const PgRel *rel, uint32 blkno, int32 mapbits, + int32 *result, PgPgError *error) +{ + bool stored; + uint64 bytes; + + if (!doltgres_relation_stored(caps, rel, &stored, error)) + return false; + if (!stored) + { + *result = mapbits; + return true; + } + if (!doltgres_entries_doltgres_relation_size(caps, rel, MAIN_FORKNUM, &bytes, error)) + return false; + *result = blkno < (bytes + (BLCKSZ - 1)) / BLCKSZ ? VISIBILITYMAP_VALID_BITS : 0; + return true; +} + +/* + * doltgres_entries_scan_frame_t is a scan cost's state across its pad: what + * it opened, the index and its table, and the table's recorded base, each + * while live, and the table's shape once read. + */ +typedef struct doltgres_entries_scan_frame_t +{ + doltgres_entries_opened_t opened; + PgRefRel index; + bool index_live; + PgRefRel table; + bool table_live; + TxnBaseBase recorded; + bool recorded_live; + const PgCatalogCaps *caps; + HostIoIo io; + + /* + * indexoid is the index a scan reads, or 0 for a bitmap scan's table + * alone. + */ + uint32 indexoid; + uint32 tableoid; + /* stored is whether the fork stores the table, and so answers its costs. */ + bool stored; + + /* + * keyed is whether the index is the table's primary key's, whose leaves + * are the table's rows. + */ + bool keyed; + + /* + * rows and tuple_ids are the blocks of the table's tree and of its tuple + * id tree. + */ + TxnBaseBlockCounts rows; + TxnBaseBlockCounts tuple_ids; + PgPgError *error; + bool ok; +} doltgres_entries_scan_frame_t; + +/* doltgres_entries_scan_pad frees what a scan cost holds. */ +static void +doltgres_entries_scan_pad(void *frame_) +{ + doltgres_entries_scan_frame_t *frame = frame_; + + if (frame->recorded_live) + txn_base_Base_drop(&frame->recorded); + if (frame->table_live) + pg_RefRel_drop(&frame->table); + if (frame->index_live) + pg_RefRel_drop(&frame->index); + frame->recorded_live = frame->table_live = frame->index_live = false; + doltgres_entries_opened_drop(&frame->opened); +} + +/* doltgres_entries_scan_opened opens a relation the planner holds a lock on, as a fork error when it is gone. */ +static bool +doltgres_entries_scan_opened(doltgres_entries_scan_frame_t *frame, uint32 relid, PgRefRel *result, bool *live) +{ + bool has; + + if (!pg_RefRel_opened(pg_CatalogCaps_may_raise(frame->caps), pg_CatalogCaps_catalog(frame->caps), relid, &has, + result, frame->error)) + return false; + if (!has) + { + *frame->error = doltgres_entries_pg_fork("XX000", "a relation the planner costs is gone"); + return false; + } + *live = true; + return true; +} + +/* + * doltgres_entries_scan_bytes returns entries times a size each plus more + * bytes, saturating: a tuple id tree's bytes, whose entries are a tuple id's + * six bytes keying the row's key and its four bytes of heap length. + */ +static uint64 +doltgres_entries_scan_bytes(uint64 entries, uint64 each, uint64 more) +{ + uint64 product; + uint64 sum; + + if (pg_mul_u64_overflow(entries, each, &product) || pg_add_u64_overflow(product, more, &sum)) + return PG_UINT64_MAX; + return sum; +} + +/* + * doltgres_entries_scan_body reads a scanned table's shape: whether the fork + * stores it, the blocks of its tree and of its tuple id tree from its + * storage's totals, and whether the index is its primary key's. + */ +static void +doltgres_entries_scan_body(void *frame_) +{ + doltgres_entries_scan_frame_t *frame = frame_; + const PgMayRaise *raise = pg_CatalogCaps_may_raise(frame->caps); + const PgRel *table; + const RepoRepo *repo; + TxnStateFound found; + TxnBaseSizes totals; + TxnBaseSizes tuple_ids = {0}; + HostError failed; + + if (frame->indexoid != 0) + { + if (!doltgres_entries_scan_opened(frame, frame->indexoid, &frame->index, &frame->index_live)) + return; + frame->tableoid = pg_calls_index_table(pg_RefRel_rel(&frame->index)); + } + if (!doltgres_entries_scan_opened(frame, frame->tableoid, &frame->table, &frame->table_live)) + return; + table = pg_RefRel_rel(&frame->table); + if (!doltgres_relation_stored(frame->caps, table, &frame->stored, frame->error)) + return; + if (!frame->stored) + { + frame->ok = true; + return; + } + frame->io = pghost_io_catalog(frame->caps); + if (!doltgres_entries_open(&frame->io, &frame->opened, frame->error)) + return; + repo = doltgres_entries_opened_repo(&frame->opened); + if (!doltgres_entries_counted(frame->caps, &frame->io, repo, &frame->opened.reading, table, table, false, + frame->error) || + !doltgres_entries_making(frame->caps, &frame->io, table, &found, frame->error)) + return; + if (!txn_state_held_Sizes(&frame->io, repo, &frame->opened.reading, found, + doltgres_entries_doltgres_index_blocks_closure_1, NULL, NULL, &totals, &failed)) + { + *frame->error = pghost_raised(&failed); + return; + } + frame->rows = txn_base_Sizes_counts(&totals, TREES_CHUNKER_DEFAULT_TARGET); + tuple_ids.entries = totals.entries; + tuple_ids.key_bytes = doltgres_entries_scan_bytes(totals.entries, 6, 0); + tuple_ids.leaf_bytes = doltgres_entries_scan_bytes(totals.entries, 10, totals.key_bytes); + frame->tuple_ids = txn_base_Sizes_counts(&tuple_ids, TREES_CHUNKER_DEFAULT_TARGET); + if (frame->indexoid != 0) + { + if (!doltgres_entries_recorded(frame->caps, &frame->io, repo, &frame->opened.reading, table, + &frame->recorded, frame->error)) + return; + frame->recorded_live = true; + if (!doltgres_index_keyed(raise, pg_CatalogCaps_catalog(frame->caps), pg_RefRel_rel(&frame->index), + &frame->recorded.key, &frame->keyed, frame->error)) + return; + } + frame->ok = true; +} + +/* + * doltgres_entries_scanned reads a scanned table's shape for its scans' + * costs, by the index it is scanned through, or by the table itself when the + * index is 0. + */ +bool +doltgres_entries_scanned(const PgCatalogCaps *caps, uint32 indexoid, uint32 tableoid, DoltgresEntriesScanned *result, + PgPgError *error) +{ + doltgres_entries_scan_frame_t frame = {.caps = caps, .indexoid = indexoid, .tableoid = tableoid, .error = error}; + + std_landing(&frame, doltgres_entries_scan_body, doltgres_entries_scan_pad); + doltgres_entries_scan_pad(&frame); + if (!frame.ok) + return false; + result->stored = frame.stored; + result->keyed = frame.keyed; + result->rows = frame.rows; + result->tuple_ids = frame.tuple_ids; + return true; +} + +/* doltgres_entries_cached_leaves returns the node cache's size in leaves of the default block size. */ +static float8 +doltgres_entries_cached_leaves(void) +{ + return (float8) pg_calls_settings_node_cache_size() * 1024.0 / BLCKSZ; +} + +/* + * doltgres_entries_leaves_read is Mackert and Lohman's count of a tree's + * distinct leaves that fetches of its rows read past the node cache, as + * index_pages_fetched counts heap pages with the node cache in + * effective_cache_size's place, and none when the tree's leaves fit the + * cache, which then holds them. + */ +static float8 +doltgres_entries_leaves_read(float8 fetches, uint64 tree_leaves, float8 cached) +{ + float8 leaves = Max((float8) tree_leaves, 1.0); + float8 limit; + + if (leaves <= cached) + return 0; + limit = 2.0 * leaves * cached / (2.0 * leaves - cached); + if (fetches <= limit) + return ceil(2.0 * leaves * fetches / (2.0 * leaves + fetches)); + return ceil(cached + (fetches - limit) * (leaves - cached) / leaves); +} + +/* + * doltgres_entries_doltgres_index_scan_io is a fork table's part of an index + * scan's costs (PLAN item 16): an index-only scan reads no row of the table; + * the primary key's own index reads its leaves, the table's rows, in key + * order, the first at random and the rest in sequence, when they don't fit + * the node cache; and a secondary index's entry leads to a descent of the + * table's tree by its row key, a keyless table's by the row's hash, its + * levels above the leaves held by the node cache at a comparison each and + * its leaf read at random when the leaves don't fit the cache. It leaves + * heap's costs for a table the fork doesn't store. + */ +bool +doltgres_entries_doltgres_index_scan_io(const PgCatalogCaps *caps, uint32 indexoid, bool indexonly, + float8 tuples_fetched, float8 selectivity, float8 loop_count, + float8 random_page_cost, float8 seq_page_cost, float8 cpu_operator_cost, + float8 *max_io, float8 *min_io, float8 *rand_heap_pages, PgPgError *error) +{ + DoltgresEntriesScanned frame; + float8 cached = doltgres_entries_cached_leaves(); + float8 read; + float8 io; + + if (!doltgres_entries_scanned(caps, indexoid, 0, &frame, error)) + return false; + if (!frame.stored) + return true; + if (indexonly) + read = io = 0; + else if (frame.keyed) + { + read = ceil(selectivity * Max((float8) frame.rows.leaves, 1.0)); + if ((float8) frame.rows.leaves <= cached) + read = io = 0; + else if (loop_count > 1) + { + read = doltgres_entries_leaves_read(read * loop_count, frame.rows.leaves, cached); + io = read * random_page_cost / loop_count; + } + else + io = read > 0 ? random_page_cost + (read - 1) * seq_page_cost : 0; + } + else + { + read = doltgres_entries_leaves_read(tuples_fetched * loop_count, frame.rows.leaves, cached); + io = tuples_fetched * frame.rows.height * cpu_operator_cost + read * random_page_cost / loop_count; + } + *max_io = *min_io = io; + *rand_heap_pages = read; + return true; +} + +/* + * doltgres_entries_doltgres_bitmap_page_cost is a fork table's cost per page + * fetched of a bitmap heap scan (PLAN item 16): each row costs its tuple id's + * resolution, a descent of the tuple id tree, and a descent of the table's + * tree by its row key, each level above the leaves a comparison and each + * leaf past the node cache read at random, and each lossy block, split from + * the pages it fetches by the bitmap's entries as compute_bitmap_pages splits + * them, costs a range scan of the tuple id tree: a descent and its leaf. It + * leaves heap's cost for a table the fork doesn't store. + */ +bool +doltgres_entries_doltgres_bitmap_page_cost(const PgCatalogCaps *caps, uint32 relid, float8 tuples_fetched, + float8 pages_fetched, uint32 pages, float8 maxentries, + float8 random_page_cost, float8 cpu_operator_cost, float8 *cost_per_page, + PgPgError *error) +{ + DoltgresEntriesScanned frame; + float8 cached = doltgres_entries_cached_leaves(); + float8 heap_pages = Min(pages_fetched, (float8) pages); + float8 lossy = 0; + float8 io; + + if (!doltgres_entries_scanned(caps, 0, relid, &frame, error)) + return false; + if (!frame.stored) + return true; + if (maxentries < heap_pages) + lossy = Max(0, heap_pages - maxentries / 2); + io = tuples_fetched * (frame.tuple_ids.height + frame.rows.height) * cpu_operator_cost + + (doltgres_entries_leaves_read(tuples_fetched, frame.tuple_ids.leaves, cached) + + doltgres_entries_leaves_read(tuples_fetched, frame.rows.leaves, cached)) * random_page_cost + + lossy * (frame.tuple_ids.height * cpu_operator_cost + + ((float8) frame.tuple_ids.leaves > cached ? random_page_cost : 0)); + *cost_per_page = pages_fetched > 0 ? io / pages_fetched : 0; + return true; +} + /* * doltgres_entries_doltgres_database_size_frame_t is database_size's state * across its pad: a directory and the tablespaces, each while live. @@ -3179,6 +3586,9 @@ typedef struct doltgres_entries_doltgres_tablespace_size_frame_t bool ok; } doltgres_entries_doltgres_tablespace_size_frame_t; +/* DOLTGRES_ENTRIES_EMPTIED_NAP is the milliseconds DROP TABLESPACE sleeps between its asks to empty a tablespace. */ +#define DOLTGRES_ENTRIES_EMPTIED_NAP 1000 + /* doltgres_entries_doltgres_tablespace_size_pad frees what tablespace_size holds. */ static void doltgres_entries_doltgres_tablespace_size_pad(void *frame_) @@ -3227,6 +3637,249 @@ doltgres_entries_doltgres_tablespace_size(const PgCaps *caps, uint32 tblspcOid, return frame.ok; } +/* + * doltgres_entries_doltgres_tablespace_emptied_frame_t is tablespace_emptied's + * state across its pad: the tablespace's directory of repositories, a path, + * and the names listed in each, while live. + */ +typedef struct doltgres_entries_doltgres_tablespace_emptied_frame_t +{ + StdPathBuf versioned; + bool versioned_live; + StdPathBuf path; + bool path_live; + StdVec names; + bool names_live; + StdVec inner; + bool inner_live; + const PgCaps *caps; + HostIoIo io; + uint32 tablespace; + bool redo; + PgPgError *error; + bool ok; +} doltgres_entries_doltgres_tablespace_emptied_frame_t; + +/* doltgres_entries_doltgres_tablespace_emptied_pad frees what tablespace_emptied holds. */ +static void +doltgres_entries_doltgres_tablespace_emptied_pad(void *frame_) +{ + doltgres_entries_doltgres_tablespace_emptied_frame_t *frame = frame_; + + if (frame->inner_live) + std_vec_drop_Checked(&frame->inner, sizeof(StdString), _Alignof(StdString), std_String_drop); + if (frame->names_live) + std_vec_drop_Checked(&frame->names, sizeof(StdString), _Alignof(StdString), std_String_drop); + if (frame->path_live) + std_vec_free_Checked(&frame->path.bytes, 1, 1); + if (frame->versioned_live) + std_vec_free_Checked(&frame->versioned.bytes, 1, 1); + frame->inner_live = frame->names_live = frame->path_live = frame->versioned_live = false; +} + +/* + * doltgres_entries_tablespace_held reports whether a directory of the + * tablespace's version directory other than the fork's holds a file, as + * Postgres' removal of its directories finds a relation's file there. + */ +static bool +doltgres_entries_tablespace_held(doltgres_entries_doltgres_tablespace_emptied_frame_t *frame, bool *result, + HostError *failed) +{ + StdStr version = host_directory_of((const char *) frame->versioned.bytes.ptr, frame->versioned.bytes.len); + bool has; + + *result = false; + if (!host_list_dir_if_there(HOST_IO_IO_READ_DOORS(&frame->io), version.ptr, version.len, &has, &frame->names, + failed)) + return false; + if (!has) + return true; + frame->names_live = true; + for (size_t k = 0, k_len = frame->names.len; k < k_len && !*result; k++) + { + const StdString *name = &STD_VEC_AT(&frame->names, StdString, k); + bool listed; + + if (name->len == strlen(HOST_VERSIONED) && memcmp(name->ptr, HOST_VERSIONED, name->len) == 0) + continue; + std_path_join(version.ptr, version.len, (const char *) name->ptr, name->len, &frame->path); + frame->path_live = true; + if (!host_list_dir_if_there(HOST_IO_IO_READ_DOORS(&frame->io), (const char *) frame->path.bytes.ptr, + frame->path.bytes.len, &listed, &frame->inner, failed)) + return false; + if (listed) + { + frame->inner_live = true; + *result = frame->inner.len != 0; + frame->inner_live = false; + std_vec_drop_Checked(&frame->inner, sizeof(StdString), _Alignof(StdString), std_String_drop); + } + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + } + frame->names_live = false; + std_vec_drop_Checked(&frame->names, sizeof(StdString), _Alignof(StdString), std_String_drop); + return true; +} + +/* + * doltgres_entries_tablespace_asked answers whether a repository's directory + * is in the tablespace's directory of repositories, asking the collector + * worker first to empty the tablespace out of each such repository when + * asking, and removes the directory of repositories once none is. + */ +static bool +doltgres_entries_tablespace_asked(doltgres_entries_doltgres_tablespace_emptied_frame_t *frame, bool asking, + bool *result, HostError *failed) +{ + bool has; + bool removed; + + *result = false; + if (!host_list_dir_if_there(HOST_IO_IO_READ_DOORS(&frame->io), (const char *) frame->versioned.bytes.ptr, + frame->versioned.bytes.len, &has, &frame->names, failed)) + return false; + if (!has) + return true; + frame->names_live = true; + for (size_t k = 0, k_len = frame->names.len; k < k_len; k++) + { + const StdString *name = &STD_VEC_AT(&frame->names, StdString, k); + uint32 repository; + uint64 counts[3]; + StdParseIntError parsed; + + if (!std_str_parse_u32(name->ptr, name->len, &repository, &parsed) || repository == 0) + continue; + *result = true; + if (!asking) + continue; + if (!host_collection_ask(&frame->io, repository, frame->tablespace, counts, failed)) + return false; + if (counts[0] == PG_UINT64_MAX) + { + StdFmtPiece pieces[] = {STD_FMT_LIT("the collector's collection of database "), STD_FMT_ARG(0), + STD_FMT_LIT(" failed")}; + StdFmtArg args[] = {{&repository, std_u32_Display_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + StdString message; + + std_fmt_format_Checked(&message, &arguments); + *failed = host_Error_fork((const uint8 *) "XX000", &message); + return false; + } + } + frame->names_live = false; + std_vec_drop_Checked(&frame->names, sizeof(StdString), _Alignof(StdString), std_String_drop); + if (*result) + return true; + return host_remove_tree(HOST_IO_IO_IO_DOORS(&frame->io), (const char *) frame->versioned.bytes.ptr, + frame->versioned.bytes.len, &removed, failed); +} + +/* + * doltgres_entries_tablespace_marked leaves the dropped mark, durably, in the + * tablespace's directory of repositories when redo's attempt left it there. + */ +static bool +doltgres_entries_tablespace_marked(doltgres_entries_doltgres_tablespace_emptied_frame_t *frame, HostError *failed) +{ + bool there; + bool made; + HostFile mark; + + if (!host_list_dir_if_there(HOST_IO_IO_READ_DOORS(&frame->io), (const char *) frame->versioned.bytes.ptr, + frame->versioned.bytes.len, &there, &frame->names, failed)) + return false; + if (!there) + return true; + std_vec_drop_Checked(&frame->names, sizeof(StdString), _Alignof(StdString), std_String_drop); + std_path_join((const char *) frame->versioned.bytes.ptr, frame->versioned.bytes.len, DOLTGRES_DROPPED_MARK, + strlen(DOLTGRES_DROPPED_MARK), &frame->path); + frame->path_live = true; + if (!host_create_if_absent(HOST_IO_IO_IO_DOORS(&frame->io), (const char *) frame->path.bytes.ptr, + frame->path.bytes.len, &made, &mark, failed)) + return false; + if (made) + host_File_drop(&mark); + return host_sync_path(HOST_IO_IO_IO_DOORS(&frame->io), (const char *) frame->versioned.bytes.ptr, + frame->versioned.bytes.len, true, failed); +} + +/* + * doltgres_entries_doltgres_tablespace_emptied_body empties the fork's packs + * out of the tablespace, round by round until its directory of repositories + * is gone or a relation's file appears in Postgres' directories there, for + * Postgres to refuse the DROP, or once in redo, which then leaves the dropped + * mark on what is left for the collector's sweep. + */ +static void +doltgres_entries_doltgres_tablespace_emptied_body(void *frame_) +{ + doltgres_entries_doltgres_tablespace_emptied_frame_t *frame = frame_; + const PgMayRaise *raise = pg_Caps_may_raise(frame->caps); + HostError failed; + bool held; + bool left; + + frame->versioned = host_versioned_dir(frame->tablespace); + frame->versioned_live = true; + if (!doltgres_entries_tablespace_held(frame, &held, &failed)) + goto raised; + if (held && !frame->redo) + { + if (!pg_calls_checkpoint_now(raise, frame->error)) + return; + if (!doltgres_entries_tablespace_held(frame, &held, &failed)) + goto raised; + } + for (bool asking = !held && (!frame->redo || host_collector_running()); asking;) + { + if (!doltgres_entries_tablespace_asked(frame, true, &left, &failed) || + (left && !doltgres_entries_tablespace_asked(frame, false, &left, &failed))) + goto raised; + if (!left || frame->redo) + break; + if (!pg_calls_latch_sleep(raise, DOLTGRES_ENTRIES_EMPTIED_NAP, frame->error)) + return; + if (!doltgres_entries_tablespace_held(frame, &held, &failed)) + goto raised; + asking = !held; + } + if (frame->redo && !doltgres_entries_tablespace_marked(frame, &failed)) + goto raised; + frame->ok = true; + return; +raised: + *frame->error = pghost_raised(&failed); +} + +/* + * doltgres_entries_doltgres_tablespace_emptied empties the fork's packs out of + * a tablespace and removes its directory of them before Postgres removes the + * tablespace's directories (design item 5, bugs row 99): when no directory of + * Postgres' there holds a file, after the checkpoint that removes the files + * of dropped relations, the collector worker copies every live chunk of + * each repository's packs there into the repository's own directory and + * retires them, the files go once their readers end, and a base backup + * keeps the collection from running, so DROP TABLESPACE waits for each, + * answering cancel. Redo makes one attempt, and none when no collector + * worker runs to answer, leaving the dropped mark on what is left, which + * the worker's sweep removes. + */ +bool +doltgres_entries_doltgres_tablespace_emptied(const PgCaps *caps, uint32 tablespaceoid, bool redo, PgPgError *error) +{ + doltgres_entries_doltgres_tablespace_emptied_frame_t frame = {.caps = caps, .io = pghost_io(pg_Caps_may_raise(caps)), + .tablespace = tablespaceoid, .redo = redo, .error = error}; + + std_landing(&frame, doltgres_entries_doltgres_tablespace_emptied_body, + doltgres_entries_doltgres_tablespace_emptied_pad); + doltgres_entries_doltgres_tablespace_emptied_pad(&frame); + return frame.ok; +} + /* * doltgres_entries_doltgres_relation_estimate_size is the fork's * relation_estimate_size: Postgres' block-based estimate over the fork's @@ -4133,7 +4786,7 @@ doltgres_entries_doltgres_parallel_based_body(void *frame_) frame->base_live = true; base = &frame->base; parts.fresh = base->fresh_end; - parts.folded = base->folded; + parts.folded = base->read; } doltgres_scan_Epochs_based(base, &epoch, &has_chain, &chain); store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, &frame->opened.reading); @@ -5093,17 +5746,20 @@ doltgres_entries_entry_pushed(HostIoIo *io, const RepoRepo *repo, const HostShar /* * doltgres_entries_index_read_captures_t is an index read's lent calls' - * captures: the Dirty snapshot it reads under and the scan reader it steps. + * captures: the Dirty snapshot it reads under, the base generation its base + * was read at, and the scan reader it steps. */ typedef struct doltgres_entries_index_read_captures_t { const HostSnapshot *read; + uint64 generation; HostOwnedHandle reader; } doltgres_entries_index_read_captures_t; /* * doltgres_entries_index_read_readable is an index read's first lent call's - * body: the base the Dirty snapshot may read. + * body: the base the Dirty snapshot may read, with the generation read + * before it. */ static bool doltgres_entries_index_read_readable(void *captures, HostIoIo *io, const RepoRepo *repo, const HostSharedHeld *held, @@ -5111,6 +5767,7 @@ doltgres_entries_index_read_readable(void *captures, HostIoIo *io, const RepoRep { doltgres_entries_index_read_captures_t *c = captures; + c->generation = txn_state_generation(host_shared_Held_record(held)); return txn_state_readable(io, held, c->read, some, result, error); } @@ -5224,8 +5881,8 @@ doltgres_entries_index_read_begin(doltgres_entries_index_read_frame_t *frame, co std_slice_to_vec_Checked(&keyed, frame->keyed, frame->keyed_len, 1, 1); frame->order_live = false; if (!doltgres_scan_scan_begin(frame->io, TREES_STORE_READER_NODE_STORE(&store), 0, some ? &frame->base : NULL, - &keyed, pg_calls_index_table(frame->index), &frame->order, &frame->captures.reader, - &failed) || + frame->captures.generation, &keyed, pg_calls_index_table(frame->index), + &frame->order, &frame->captures.reader, &failed) || !doltgres_scan_seek(frame->io, &stores, frame->captures.reader, from, from_len, DOLTGRES_SCAN_DIRECTION_FORWARD, &failed)) { @@ -6435,10 +7092,6 @@ doltgres_entries_pushed_closure_0(void *captures, HostIoIo *io, const RepoRepo * doltgres_entries_pushed_captures_t *c = captures; const doltgres_entries_Ends *ends = c->ends; HostIoWaits waits = pghost_waits(); - bool has_newest; - TxnBaseBase newest; - bool has_tuple_ids = false; - TreesStoreTree tuple_ids = {0}; bool has_key; bool keyless; TxnChangeEnding row; @@ -6446,16 +7099,8 @@ doltgres_entries_pushed_closure_0(void *captures, HostIoIo *io, const RepoRepo * c->store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); c->held = held; - if (!txn_state_newest(io, held, &has_newest, &newest, error)) - return false; - if (has_newest) - { - has_tuple_ids = newest.has_tuple_ids; - tuple_ids = newest.tuple_ids; - txn_base_Base_drop(&newest); - } - if (!doltgres_scan_keyed(io, TREES_STORE_READER_NODE_STORE(&c->store), host_shared_Held_record(held), - has_tuple_ids, tuple_ids, c->tuple, &has_key, &row.key, &keyless, error)) + if (!doltgres_scan_keyed(io, TREES_STORE_READER_NODE_STORE(&c->store), held, c->tuple, &has_key, &row.key, + &keyless, error)) return false; *some = has_key; if (!has_key) @@ -7273,10 +7918,6 @@ doltgres_entries_doltgres_tuple_update_closure_1_body(void *frame_) doltgres_entries_doltgres_tuple_update_captures_t *c = frame->c; const doltgres_entries_Written *w = c->written; HostStorage storage = doltgres_entries_storage(c->rel); - bool has_newest; - TxnBaseBase newest; - bool has_tuple_ids = false; - TreesStoreTree tuple_ids = {0}; bool has_key; bool keyless; TxnInsertKeyed keyed = {.stripe = w->stripe}; @@ -7284,16 +7925,7 @@ doltgres_entries_doltgres_tuple_update_closure_1_body(void *frame_) c->store = repo_Repo_reader(frame->repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); c->held = frame->held; - if (!txn_state_newest(frame->io, frame->held, &has_newest, &newest, frame->error)) - return; - if (has_newest) - { - has_tuple_ids = newest.has_tuple_ids; - tuple_ids = newest.tuple_ids; - txn_base_Base_drop(&newest); - } - if (!doltgres_scan_keyed(frame->io, TREES_STORE_READER_NODE_STORE(&c->store), - host_shared_Held_record(frame->held), has_tuple_ids, tuple_ids, c->tuple, &has_key, + if (!doltgres_scan_keyed(frame->io, TREES_STORE_READER_NODE_STORE(&c->store), frame->held, c->tuple, &has_key, &frame->replaces.key, &keyless, frame->error)) return; *frame->some = has_key; @@ -8113,9 +8745,21 @@ doltgres_entries_build_scanned(const PgCatalogCaps *caps, HostIoIo *io, const Re #define TXN_STATE_DECLARE #include "doltgres/txn/state_template.h" +/* + * doltgres_entries_doltgres_index_build_range_scan_captures_t is the build + * scan's lent call's captures: the table, and the base generation its base + * was read at. + */ +typedef struct doltgres_entries_doltgres_index_build_range_scan_captures_t +{ + const PgRel *table; + uint64 generation; +} doltgres_entries_doltgres_index_build_range_scan_captures_t; + /* * doltgres_entries_doltgres_index_build_range_scan_closure_0 is the build - * scan's lent call: the table's blocks and newest base. + * scan's lent call: the table's blocks and newest base, with the generation + * read before it. */ static bool doltgres_entries_doltgres_index_build_range_scan_closure_0(void *captures, HostIoIo *io, const RepoRepo *repo, @@ -8123,7 +8767,10 @@ doltgres_entries_doltgres_index_build_range_scan_closure_0(void *captures, HostI bool *has_result_1, TxnBaseBase *result_1, HostError *error) { - *result_0 = doltgres_entries_blocks(captures, held); + doltgres_entries_doltgres_index_build_range_scan_captures_t *c = captures; + + *result_0 = doltgres_entries_blocks(c->table, held); + c->generation = txn_state_generation(host_shared_Held_record(held)); return txn_state_newest(io, held, has_result_1, result_1, error); } @@ -8209,6 +8856,7 @@ doltgres_entries_doltgres_index_build_range_scan_own(doltgres_entries_doltgres_i TreesStoreReader store; uint64 last; HostError failed; + doltgres_entries_doltgres_index_build_range_scan_captures_t c = {frame->table}; if (!doltgres_entries_recorded(frame->caps, &frame->io, repo, &frame->opened.reading, frame->table, &frame->recorded, frame->error)) @@ -8227,9 +8875,8 @@ doltgres_entries_doltgres_index_build_range_scan_own(doltgres_entries_doltgres_i if (!doltgres_entries_making(frame->caps, &frame->io, frame->table, &found, frame->error)) return false; if (!txn_state_held_TupleU64OptionBase(&frame->io, repo, &frame->opened.reading, found, - doltgres_entries_doltgres_index_build_range_scan_closure_0, - (void *) frame->table, NULL, blocks, &frame->base_live, &frame->base, - &failed)) + doltgres_entries_doltgres_index_build_range_scan_closure_0, &c, NULL, + blocks, &frame->base_live, &frame->base, &failed)) { frame->base_live = false; *frame->error = pghost_raised(&failed); @@ -8238,7 +8885,7 @@ doltgres_entries_doltgres_index_build_range_scan_own(doltgres_entries_doltgres_i store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, &frame->opened.reading); frame->keyed_live = frame->order_live = false; if (!doltgres_scan_scan_begin(&frame->io, TREES_STORE_READER_NODE_STORE(&store), *blocks, - frame->base_live ? &frame->base : NULL, &frame->keyed, + frame->base_live ? &frame->base : NULL, c.generation, &frame->keyed, pg_calls_rel_id(frame->table), &frame->order, handle, &failed)) { *frame->error = pghost_raised(&failed); @@ -8359,6 +9006,169 @@ doltgres_entries_doltgres_index_build_range_scan(const PgCatalogCaps *caps, cons return frame.ok; } +/* + * doltgres_entries_BuildSnapshot is a concurrent build's snapshot, which its + * validation reads in the same backend: the index built, and the snapshot's + * bounds, whether its subtransactions overflowed, and its running xids, each + * a Vec. + */ +typedef struct doltgres_entries_BuildSnapshot +{ + uint32 index; + uint64 xmin; + uint64 xmax; + bool overflowed; + StdVec xip; + StdVec subxip; +} doltgres_entries_BuildSnapshot; + +/* doltgres_entries_BuildSnapshot_drop is BuildSnapshot's drop: its running xids. */ +static void +doltgres_entries_BuildSnapshot_drop(void *snapshot) +{ + doltgres_entries_BuildSnapshot *self = snapshot; + + std_vec_free_Checked(&self->xip, sizeof(uint32), _Alignof(uint32)); + std_vec_free_Checked(&self->subxip, sizeof(uint32), _Alignof(uint32)); +} + +/* doltgres_entries_VecBuildSnapshot_drop is Vec's drop. */ +static void +doltgres_entries_VecBuildSnapshot_drop(void *snapshots) +{ + std_vec_drop_Checked(snapshots, sizeof(doltgres_entries_BuildSnapshot), _Alignof(doltgres_entries_BuildSnapshot), + doltgres_entries_BuildSnapshot_drop); +} + +/* BUILD_SNAPSHOTS' IoMutex, host's lock at the backend's concurrent builds' snapshots. */ +#define HOST_LOCK_SCOPE static +#define HOST_LOCK_DECLARE +#define HOST_LOCK_DEFINE +#define HOST_LOCK_TAKEN_TS VecBuildSnapshot +#define HOST_LOCK_TAKEN_T StdVec +#define HOST_LOCK_TAKEN_DEFAULT(place) \ + std_vec_new((place), sizeof(doltgres_entries_BuildSnapshot), _Alignof(doltgres_entries_BuildSnapshot)) +#define HOST_LOCK_TAKEN_DROP(place) doltgres_entries_VecBuildSnapshot_drop(place) +#define HOST_LOCK_TAKEN_TYPE DOLTGRES_ENTRIES_BUILD_SNAPSHOTS_LOCK_TYPE +#define HOST_LOCK_IO_MUTEX +#define HOST_LOCK_TS VecBuildSnapshot +#define HOST_LOCK_T StdVec +#define HOST_LOCK_DEFAULT(place) \ + std_vec_new((place), sizeof(doltgres_entries_BuildSnapshot), _Alignof(doltgres_entries_BuildSnapshot)) +#include "doltgres/host/lock_template.h" + +/* doltgres_entries_IoMutexVecBuildSnapshot_drop is IoMutex>' drop glue: its value's. */ +static void +doltgres_entries_IoMutexVecBuildSnapshot_drop(void *value) +{ + doltgres_entries_VecBuildSnapshot_drop(&((HostLockIoMutexVecBuildSnapshot *) value)->value); +} + +/* DOLTGRES_ENTRIES_BUILD_SNAPSHOTS_TYPE is IoMutex>' size, Default, and drop. */ +static const StdDefaultType DOLTGRES_ENTRIES_BUILD_SNAPSHOTS_TYPE = {sizeof(HostLockIoMutexVecBuildSnapshot), + _Alignof(HostLockIoMutexVecBuildSnapshot), host_lock_IoMutexVecBuildSnapshot_Default_default, +doltgres_entries_IoMutexVecBuildSnapshot_drop}; + +/* + * DOLTGRES_ENTRIES_BUILD_SNAPSHOTS is BUILD_SNAPSHOTS: each concurrent build's + * snapshot from its build's end until its validation takes it, in backend + * memory alone (bugs row 313); its BackendLocal header and the process's + * value. + */ +static struct +{ + HostLocalBackendLocal local; + HostLockIoMutexVecBuildSnapshot process; +} DOLTGRES_ENTRIES_BUILD_SNAPSHOTS = {HOST_LOCAL_BACKEND_LOCAL_NEW}; + +/* doltgres_entries_build_snapshot_kept_op moves a build's snapshot into BUILD_SNAPSHOTS, in place of its index's older one. */ +static void +doltgres_entries_build_snapshot_kept_op(void *value, void *captures) +{ + StdVec *self = value; + doltgres_entries_BuildSnapshot *kept = captures; + + for (size_t i = 0; i < self->len; i++) + { + doltgres_entries_BuildSnapshot *at = &STD_VEC_AT(self, doltgres_entries_BuildSnapshot, i); + + if (at->index == kept->index) + { + doltgres_entries_BuildSnapshot_drop(at); + *at = *kept; + return; + } + } + std_vec_push_Checked(self, kept, sizeof(*kept), _Alignof(doltgres_entries_BuildSnapshot)); +} + +/* doltgres_entries_build_snapshot_taken_t is a take of BUILD_SNAPSHOTS' captures: the index, and its snapshot when it had one. */ +typedef struct doltgres_entries_build_snapshot_taken_t +{ + uint32 index; + bool has; + doltgres_entries_BuildSnapshot snapshot; +} doltgres_entries_build_snapshot_taken_t; + +/* doltgres_entries_build_snapshot_taken_op takes an index's build snapshot out of BUILD_SNAPSHOTS into its captures. */ +static void +doltgres_entries_build_snapshot_taken_op(void *value, void *captures) +{ + StdVec *self = value; + doltgres_entries_build_snapshot_taken_t *taken = captures; + + for (size_t i = 0; i < self->len; i++) + if (STD_VEC_AT(self, doltgres_entries_BuildSnapshot, i).index == taken->index) + { + std_vec_swap_remove(self, i, &taken->snapshot, sizeof(doltgres_entries_BuildSnapshot), + _Alignof(doltgres_entries_BuildSnapshot)); + taken->has = true; + return; + } +} + +/* + * doltgres_entries_build_snapshot_copied copies the transaction's snapshot + * for a concurrent build of an index, taken before the build reads a row, as + * heap's concurrent build scan takes its own. + */ +static bool +doltgres_entries_build_snapshot_copied(const PgMayRaise *raise, uint32 index, doltgres_entries_BuildSnapshot *result, + PgPgError *error) +{ + PgSnapshotView view; + + if (!pg_transaction_snapshot_view(raise, &view, error)) + return false; + result->index = index; + result->xmin = view.xmin; + result->xmax = view.xmax; + result->overflowed = view.overflowed; + std_slice_to_vec_Checked(&result->xip, view.xip, view.xip_len, sizeof(uint32), _Alignof(uint32)); + std_slice_to_vec_Checked(&result->subxip, view.subxip, view.subxip_len, sizeof(uint32), _Alignof(uint32)); + return true; +} + +/* + * doltgres_entries_build_snapshot_saw reports whether a concurrent build's + * snapshot saw a row's writer ended, so that the build indexed the row: an + * xid the snapshot counts as no longer running, which an overflowed snapshot + * answers only below its xmin. + */ +static bool +doltgres_entries_build_snapshot_saw(const doltgres_entries_BuildSnapshot *built, uint64 writer) +{ + HostSnapshot snapshot = {.kind = HOST_SNAPSHOT_KIND_MVCC, .xmin = {built->xmin}, .xmax = {built->xmax}, + .overflowed = built->overflowed, .xip = built->xip.ptr, .xip_len = built->xip.len, + .subxip = built->subxip.ptr, .subxip_len = built->subxip.len}; + + if (writer == InvalidTransactionId) + return false; + if (built->overflowed) + return writer < built->xmin; + return !txn_visible_in_snapshot(&snapshot, writer, writer); +} + /* doltgres_entries_doltgres_index_validate_scan_closure_0 is the validate scan's lent call: the table's block count. */ static bool doltgres_entries_doltgres_index_validate_scan_closure_0(void *captures, HostIoIo *io, const RepoRepo *repo, @@ -8387,6 +9197,7 @@ typedef struct doltgres_entries_doltgres_index_validate_scan_frame_t PgIndexInfo *info; const PgSnapshot *snapshot; PgValidateState *state; + doltgres_entries_build_snapshot_taken_t built; PgPgError *error; bool ok; } doltgres_entries_doltgres_index_validate_scan_frame_t; @@ -8401,7 +9212,9 @@ doltgres_entries_doltgres_index_validate_scan_pad(void *frame_) pg_OwnedBuild_drop(&frame->build); if (frame->slot_live) pg_OwnedSlot_drop(&frame->slot); - frame->build_live = frame->slot_live = false; + if (frame->built.has) + doltgres_entries_BuildSnapshot_drop(&frame->built.snapshot); + frame->build_live = frame->slot_live = frame->built.has = false; doltgres_entries_opened_drop(&frame->opened); } @@ -8430,6 +9243,10 @@ doltgres_entries_doltgres_index_validate_scan_body(void *frame_) HostError failed; frame->io = pghost_io_catalog(frame->caps); + frame->built.index = pg_calls_rel_id(frame->index); + doltgres_entries_locked(&frame->io, &DOLTGRES_ENTRIES_BUILD_SNAPSHOTS.local, &DOLTGRES_ENTRIES_BUILD_SNAPSHOTS_TYPE, + &DOLTGRES_ENTRIES_BUILD_SNAPSHOTS_LOCK_TYPE, doltgres_entries_build_snapshot_taken_op, + &frame->built); if (!doltgres_entries_open(&frame->io, &frame->opened, frame->error) || !doltgres_entries_making(frame->caps, &frame->io, frame->table, &found, frame->error)) return; @@ -8487,6 +9304,7 @@ doltgres_entries_doltgres_index_validate_scan_body(void *frame_) ended = !has_sorted; } if ((!has_sorted || sorted != tuple) && + !(frame->built.has && doltgres_entries_build_snapshot_saw(&frame->built.snapshot, pg_calls_slot_writer(slot))) && !pg_calls_validate_insert(raise, catalog, pg_OwnedBuild_build(&frame->build), frame->index, frame->table, frame->info, slot, &added, frame->error)) return; @@ -8513,11 +9331,13 @@ doltgres_entries_doltgres_index_validate_scan_body(void *frame_) * row whose tid the index lacks and the index's predicate admits is inserted * through the index's own aminsert, a unique index checking the row. An * index the fork stores hands the merge no tid, since its ambulkdelete - * reports none, so each row is checked against the index's own entries - * instead (index_holds, bugs row 168): VACUUM's callback, which would hand - * the merge those tids, would make every VACUUM walk the whole index for - * nothing, since no entry the fork stores names a dead row, and a keyed - * table's entries hold no tids. It reports the scan's blocks to the build's + * reports none, so each row but one the build indexed is checked against the + * index's own entries instead (index_holds, bugs row 168): VACUUM's + * callback, which would hand the merge those tids, would make every VACUUM + * walk the whole index for nothing, since no entry the fork stores names a + * dead row, and a keyed table's entries hold no tids. A row the build indexed is one whose writer + * the build's snapshot saw ended (BUILD_SNAPSHOTS), and its key is not formed + * again (bugs row 313). It reports the scan's blocks to the build's * progress, counts the rows it read and inserted in the validation's state, * and clears the index's expression state, which lived in the scan's * executor state. @@ -8765,7 +9585,7 @@ doltgres_entries_rekeyed_closure_4(void *captures, HostIoIo *io, const RepoRepo TxnBaseBase installing; TreesStoreReader store; const TxnRecordMapCarried *spaces = c->spaces->ptr; - const DoltgresRewriteCarry *carried = c->carried->ptr; + const TxnRewriteCarry *carried = c->carried->ptr; if (!txn_state_base_record(io, held, &some, &installing, error)) return false; @@ -8786,16 +9606,9 @@ doltgres_entries_rekeyed_closure_4(void *captures, HostIoIo *io, const RepoRepo return false; if (!*result) return true; - for (size_t i = 0; i < c->spaces->len; i++) - if (!txn_rewrite_spaced(io, held, &spaces[i], c->logged, pg_calls_max_heap_tuples_per_page(), false, 0, - error)) - return false; store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - for (size_t i = 0; i < c->carried->len; i++) - if (!txn_rewrite_carried(io, TREES_STORE_READER_NODE_STORE(&store), held, &carried[i].f0, carried[i].f1, - carried[i].f2, c->logged, NULL, error)) - return false; - return true; + return txn_rewrite_carried(io, TREES_STORE_READER_NODE_STORE(&store), held, spaces, c->spaces->len, carried, + c->carried->len, c->logged, NULL, pg_calls_max_heap_tuples_per_page(), error); } /* doltgres_entries_Rekeyed_carried_drop is a rekeyed version's drop glue: its key's and its op's. */ @@ -8888,7 +9701,7 @@ doltgres_entries_rekeyed_pad(void *frame_) if (frame->blobs_live) std_vec_free_Checked(&frame->blobs, sizeof(chunks_TupleAddressU64_t), _Alignof(chunks_TupleAddressU64_t)); if (frame->carried_live) - std_vec_drop_Checked(&frame->carried, sizeof(DoltgresRewriteCarry), _Alignof(DoltgresRewriteCarry), + std_vec_drop_Checked(&frame->carried, sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry), doltgres_entries_Carry_drop); if (frame->toast_live) host_shared_Held_drop(&frame->values); @@ -8970,12 +9783,12 @@ doltgres_entries_rekeyed_carried(doltgres_entries_rekeyed_frame_t *frame, HostSt _Alignof(doltgres_TupleVecU8OpWriterShownLockMode_t)); frame->versions_live = true; std_vec_with_capacity_Checked(&frame->carried, std_VecIntoIter_len(&frame->versions), - sizeof(DoltgresRewriteCarry), _Alignof(DoltgresRewriteCarry)); + sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry)); frame->carried_live = true; while (std_VecIntoIter_Iterator_next(&frame->versions, &version)) { const TxnMemtablePut *put = NULL; - DoltgresRewriteCarry carry = {0}; + TxnRewriteCarry carry = {0}; TxnRecordRowChange *change = &carry.f0; if (version.f1.tag == TXN_MEMTABLE_OP_PUT) @@ -9030,7 +9843,7 @@ doltgres_entries_rekeyed_carried(doltgres_entries_rekeyed_frame_t *frame, HostSt std_vec_new(&change->kept, sizeof(size_t), _Alignof(size_t)); carry.f1 = version.f3; carry.f2 = version.f4; - std_vec_push_Checked(&frame->carried, &carry, sizeof(DoltgresRewriteCarry), _Alignof(DoltgresRewriteCarry)); + std_vec_push_Checked(&frame->carried, &carry, sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry)); } frame->versions_live = false; std_VecIntoIter_drop(&frame->versions); @@ -10416,9 +11229,38 @@ doltgres_entries_loaded_records(doltgres_entries_loaded_frame_t *frame, bool key } } +/* doltgres_entries_loaded_logging_t is an index build's install's logging's captures: the build and its storage. */ +typedef struct doltgres_entries_loaded_logging_t +{ + const doltgres_entries_loaded_frame_t *frame; + HostStorage storage; +} doltgres_entries_loaded_logging_t; + +/* + * doltgres_entries_loaded_logged is an index build's install's logging: it + * inserts the installed base's record and gives its end. + */ +static bool +doltgres_entries_loaded_logged(void *captures, HostIoIoDoors io, HostLsn reached, HostLsn *end, HostError *error) +{ + const doltgres_entries_loaded_logging_t *logging = captures; + const doltgres_entries_loaded_frame_t *frame = logging->frame; + TxnRecordBaseAdvanced advanced = {.storage = logging->storage, .folded = frame->point, + .max_folded = frame->max_folded, .base = frame->installed, .packs = frame->packs}; + StdVec body; + bool ok; + + (void) reached; + txn_record_BaseAdvanced_encode(&advanced, &body); + ok = host_wal_insert(io, HOST_WAL_KIND_BASE_ADVANCED, body.ptr, body.len, end, error); + std_vec_free_Checked(&body, 1, 1); + return ok; +} + /* * doltgres_entries_loaded_closure_7 is loaded's register run: the tree's root - * held at a slot and published, and the base installed. + * held at a slot and published, and the base installed, a permanent + * storage's packs listed first and its record logged in the install's step. */ static bool doltgres_entries_loaded_closure_7(void *captures, HostIoRegistering *registering, ChunksPacksRootsRootSlots *slots, @@ -10429,6 +11271,11 @@ doltgres_entries_loaded_closure_7(void *captures, HostIoRegistering *registering ChunksPacksRootsRootSlot *slot; bool installed; HostError failed; + TxnStatePersistence persistence; + doltgres_entries_loaded_logging_t logging = {.frame = frame}; + bool permanent = txn_state_described(host_shared_Held_record(&frame->held), &logging.storage, + &persistence) && + persistence == TXN_STATE_PERSISTENCE_PERMANENT; chunks_packs_roots_RootSlots_one(slots, &slot); if (!trees_store_BrandedWriter_hold(&frame->writer, io, slot, frame->root, error)) @@ -10444,9 +11291,13 @@ doltgres_entries_loaded_closure_7(void *captures, HostIoRegistering *registering frame->installed.root = trees_published_Published_base(&STD_VEC_AT(&frame->published, TreesPublishedPublished, 0)); frame->installed.folded = frame->point; + frame->installed.read = frame->point; frame->installed.max_folded = frame->max_folded; frame->installed.sizes = frame->sizes; - if (!txn_state_installed(io, &frame->held, frame->generation, &frame->installed, &installed, &failed)) + if (permanent && !chunks_packs_backend_Packs_flush(&frame->repo->packs, io, frame->reading, error)) + return false; + if (!txn_state_installed(io, &frame->held, frame->generation, &frame->installed, + permanent ? doltgres_entries_loaded_logged : NULL, &logging, &installed, &failed)) { *error = chunks_error_Error_FromError_from(&failed); return false; @@ -10466,55 +11317,22 @@ doltgres_entries_loaded_closure_7(void *captures, HostIoRegistering *registering return true; } -/* doltgres_entries_loaded_advanced logs the installed base as advanced, for a permanent index storage. */ +/* doltgres_entries_loaded_flushed flushes the WAL through the point, for a permanent index storage. */ static bool -doltgres_entries_loaded_advanced(doltgres_entries_loaded_frame_t *frame) +doltgres_entries_loaded_flushed(doltgres_entries_loaded_frame_t *frame) { HostStorage storage; TxnStatePersistence persistence; - TxnRecordBaseAdvanced advanced = {0}; - StdVec body; - HostLsn lsn; - ChunksErrorError failed; - bool ok; + HostError host; if (!txn_state_described(host_shared_Held_record(&frame->held), &storage, &persistence) || persistence != TXN_STATE_PERSISTENCE_PERMANENT) return true; - if (!chunks_packs_backend_Packs_flush(&frame->repo->packs, frame->io, frame->reading, &failed)) + if (!host_wal_flush(HOST_IO_IO_IO_DOORS(frame->io), frame->point, &host)) { - doltgres_entries_loaded_failed(frame, &failed); + *frame->error = pghost_raised(&host); return false; } - { - HostError host; - - if (!host_wal_flush(HOST_IO_IO_IO_DOORS(frame->io), frame->point, &host)) - { - *frame->error = pghost_raised(&host); - return false; - } - } - advanced.storage = storage; - advanced.folded = frame->point; - advanced.max_folded = frame->max_folded; - advanced.base = frame->installed; - advanced.packs = frame->packs; - frame->installed_live = frame->packs_live = false; - txn_record_BaseAdvanced_encode(&advanced, &body); - { - HostError host; - - ok = host_wal_insert(HOST_IO_IO_IO_DOORS(frame->io), HOST_WAL_KIND_BASE_ADVANCED, body.ptr, body.len, &lsn, - &host); - std_vec_free_Checked(&body, 1, 1); - txn_record_BaseAdvanced_drop(&advanced); - if (!ok) - { - *frame->error = pghost_raised(&host); - return false; - } - } return true; } @@ -10655,17 +11473,31 @@ doltgres_entries_loaded_body(void *frame_) *frame->error = doltgres_entries_internal(message, sizeof(message) - 1); return; } - if (!doltgres_entries_loaded_advanced(frame)) + if (!doltgres_entries_loaded_flushed(frame)) return; store = repo_Repo_reader(frame->repo, HOST_CACHE_PLACEMENT_SERVER, frame->reading); - for (size_t i = 0; i < frame->late.len; i++) - if (!txn_rewrite_carried(frame->io, TREES_STORE_READER_NODE_STORE(&store), &frame->held, - &STD_VEC_AT(&frame->late, TxnRecordRowChange, i), shown, TXN_MEMTABLE_LOCK_MODE_UPDATE, - pg_calls_rel_permanent(frame->index), NULL, &host)) + { + StdVec late; + bool ok; + + std_vec_with_capacity_Checked(&late, frame->late.len, sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry)); + for (size_t i = 0; i < frame->late.len; i++) + { + TxnRewriteCarry carry = {STD_VEC_AT(&frame->late, TxnRecordRowChange, i), shown, + TXN_MEMTABLE_LOCK_MODE_UPDATE}; + + std_vec_push_Checked(&late, &carry, sizeof(carry), _Alignof(TxnRewriteCarry)); + } + ok = txn_rewrite_carried(frame->io, TREES_STORE_READER_NODE_STORE(&store), &frame->held, NULL, 0, late.ptr, + late.len, pg_calls_rel_permanent(frame->index), NULL, + pg_calls_max_heap_tuples_per_page(), &host); + std_vec_free_Checked(&late, sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry)); + if (!ok) { *frame->error = pghost_raised(&host); return; } + } *frame->result = frame->sizes.rows + late_rows; frame->ok = true; } @@ -10742,6 +11574,8 @@ typedef struct doltgres_entries_doltgres_index_build_frame_t bool described_live; PgOwnedSort sort; bool sort_live; + doltgres_entries_BuildSnapshot built; + bool built_live; const PgCatalogCaps *caps; HostIoIo io; const PgRel *heap; @@ -10767,10 +11601,24 @@ doltgres_entries_doltgres_index_build_pad(void *frame_) doltgres_layout_VecKeyed_drop(&frame->table_key); if (frame->key_live) txn_base_Key_drop(&frame->key); - frame->sort_live = frame->described_live = frame->table_key_live = frame->key_live = false; + if (frame->built_live) + doltgres_entries_BuildSnapshot_drop(&frame->built); + frame->sort_live = frame->described_live = frame->table_key_live = frame->key_live = frame->built_live = false; doltgres_entries_opened_drop(&frame->opened); } +/* doltgres_entries_doltgres_index_build_kept moves a concurrent build's snapshot into BUILD_SNAPSHOTS once the build succeeds. */ +static void +doltgres_entries_doltgres_index_build_kept(doltgres_entries_doltgres_index_build_frame_t *frame) +{ + if (!frame->ok || !frame->built_live) + return; + doltgres_entries_locked(&frame->io, &DOLTGRES_ENTRIES_BUILD_SNAPSHOTS.local, &DOLTGRES_ENTRIES_BUILD_SNAPSHOTS_TYPE, + &DOLTGRES_ENTRIES_BUILD_SNAPSHOTS_LOCK_TYPE, doltgres_entries_build_snapshot_kept_op, + &frame->built); + frame->built_live = false; +} + /* doltgres_entries_doltgres_index_build_body scans the table into the build's sort and loads the sorted entries. */ static void doltgres_entries_doltgres_index_build_body(void *frame_) @@ -10824,6 +11672,12 @@ doltgres_entries_doltgres_index_build_body(void *frame_) !pg_calls_sort_begin(raise, &frame->sort, frame->error)) return; frame->sort_live = true; + if (pg_calls_index_info_concurrent(frame->info)) + { + if (!doltgres_entries_build_snapshot_copied(raise, pg_calls_rel_id(frame->index), &frame->built, frame->error)) + return; + frame->built_live = true; + } pg_calls_progress_update(PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_BTREE_PHASE_INDEXBUILD_TABLESCAN); doltgres_entries_locked(&frame->io, &DOLTGRES_ENTRIES_BUILD_DESCRIBED.local, &DOLTGRES_ENTRIES_BUILD_DESCRIBED_TYPE, &DOLTGRES_ENTRIES_BUILD_DESCRIBED_LOCK_TYPE, doltgres_entries_describing_replaced_op, @@ -10839,6 +11693,7 @@ doltgres_entries_doltgres_index_build_body(void *frame_) if (doltgres_index_Described_keyed(described)) { frame->ok = pg_calls_build_result(raise, heap_tuples, (float8) put, frame->result, frame->error); + doltgres_entries_doltgres_index_build_kept(frame); return; } pg_calls_progress_update_three(PROGRESS_CREATEIDX_TUPLES_TOTAL, put > PG_INT64_MAX ? PG_INT64_MAX : (int64) put, @@ -10866,6 +11721,7 @@ doltgres_entries_doltgres_index_build_body(void *frame_) frame->info, &frame->sort, described, horizon, based, &entries, frame->error)) return; frame->ok = pg_calls_build_result(raise, heap_tuples, (float8) entries, frame->result, frame->error); + doltgres_entries_doltgres_index_build_kept(frame); } /* @@ -10879,7 +11735,8 @@ doltgres_entries_doltgres_index_build_body(void *frame_) * the sort as the load's total, with the scan's block counts reset in the * same update. It answers the table's live rows and the rows its entries * hold. A new index records its identity pairs first, in its creating - * transaction (design item 10). + * transaction (design item 10). A concurrent build keeps the snapshot it took + * before its scan for its validation (BUILD_SNAPSHOTS). */ bool doltgres_entries_doltgres_index_build(const PgCatalogCaps *caps, const PgRel *heap, const PgRel *index, @@ -11267,13 +12124,11 @@ doltgres_entries_doltgres_index_delete_tuples(const PgCatalogCaps *caps, const P /* * doltgres_entries_doltgres_index_holds_captures_t is index_holds' lent call's - * captures: the reader, the base's tuple ids, and the row's tuple id. + * captures: the reader and the row's tuple id. */ typedef struct doltgres_entries_doltgres_index_holds_captures_t { const HostSharedReader *reading; - bool has_tuple_ids; - TreesStoreTree tuple_ids; uint64 tuple; } doltgres_entries_doltgres_index_holds_captures_t; @@ -11286,8 +12141,8 @@ doltgres_entries_doltgres_index_holds_closure_0(void *captures, HostIoIo *io, co doltgres_entries_doltgres_index_holds_captures_t *c = captures; TreesStoreReader store = repo_Repo_reader(repo, HOST_CACHE_PLACEMENT_SERVER, c->reading); - return doltgres_scan_keyed(io, TREES_STORE_READER_NODE_STORE(&store), host_shared_Held_record(held), - c->has_tuple_ids, c->tuple_ids, c->tuple, some, result_0, result_1, error); + return doltgres_scan_keyed(io, TREES_STORE_READER_NODE_STORE(&store), held, c->tuple, some, result_0, result_1, + error); } /* @@ -11388,8 +12243,6 @@ doltgres_entries_doltgres_index_holds_body(void *frame_) return; } captures.tuple = pg_calls_tid_value(frame->heap_tid); - captures.has_tuple_ids = frame->base.has_tuple_ids; - captures.tuple_ids = frame->base.tuple_ids; if (!doltgres_entries_making(frame->caps, &frame->io, frame->heap, &made, frame->error)) return; if (!txn_state_held_OptionTupleVecU8Bool(&frame->io, repo, &frame->opened.reading, made, diff --git a/src/doltgres/doltgres/c/rewrite.c b/src/doltgres/doltgres/c/rewrite.c index 30a3588495980..f753f1264c61c 100644 --- a/src/doltgres/doltgres/c/rewrite.c +++ b/src/doltgres/doltgres/c/rewrite.c @@ -354,7 +354,7 @@ doltgres_rewrite_read_body(void *frame_) std_vec_new(&read->olds, DOLTGRES_REWRITE_OLD); frame->rest_live = true; read->left_out = read->base.max_ended >= frame->horizon ? read->base.max_ended : 0; - if (!txn_rewrite_keys(frame->io, frame->held, read->based.ptr, read->based.len, read->base.folded, + if (!txn_rewrite_keys(frame->io, frame->held, read->based.ptr, read->based.len, read->base.read, doltgres_rewrite_read_closure_0, &captures, NULL, frame->error)) return; *frame->result = *read; @@ -1646,7 +1646,7 @@ doltgres_rewrite_Reformer_reformed(DoltgresRewriteReformer *self, const PgCatalo /* More element layouts of the rewrite's vectors. */ #define DOLTGRES_REWRITE_COPIED sizeof(TxnRewriteCopied), _Alignof(TxnRewriteCopied) #define DOLTGRES_REWRITE_COPY sizeof(txn_TupleU64U64U32_t), _Alignof(txn_TupleU64U64U32_t) -#define DOLTGRES_REWRITE_CARRY sizeof(DoltgresRewriteCarry), _Alignof(DoltgresRewriteCarry) +#define DOLTGRES_REWRITE_CARRY sizeof(TxnRewriteCarry), _Alignof(TxnRewriteCarry) #define DOLTGRES_REWRITE_KEPT_ROW sizeof(doltgres_rewrite_kept_t), _Alignof(doltgres_rewrite_kept_t) #define DOLTGRES_REWRITE_ROW_ENDS sizeof(doltgres_rewrite_row_ends_t), _Alignof(doltgres_rewrite_row_ends_t) #define DOLTGRES_REWRITE_ROWS sizeof(StdVec), _Alignof(StdVec) @@ -1756,7 +1756,7 @@ doltgres_rewrite_copied(const DoltgresRewriteRead *read, DoltgresRewriteRewritte static void doltgres_rewrite_Carry_drop(void *carry) { - txn_record_RowChange_drop(&((DoltgresRewriteCarry *) carry)->f0); + txn_record_RowChange_drop(&((TxnRewriteCarry *) carry)->f0); } /* @@ -2134,7 +2134,7 @@ doltgres_rewrite_carrying_body(void *frame_) TxnMemtableVersionOpRef ref; TxnMemtableLockMode lock; const StdVec *values = NULL; - DoltgresRewriteCarry carry = {0}; + TxnRewriteCarry carry = {0}; HostRow row = {0}; StdBytes rekeyed; bool has_row = true; @@ -2331,7 +2331,7 @@ doltgres_rewrite_moving_closure_0(void *captures, HostIoIo *io, const uint8 *key TxnMemtableVersionVersion *version = &STD_VEC_AT(versions, TxnMemtableVersionVersion, i); TxnMemtableVersionRecord record; TxnMemtableVersionOpRef ref; - DoltgresRewriteCarry carry = {0}; + TxnRewriteCarry carry = {0}; HostStatus outcome; if (!txn_outcome_outcome(io, version, &outcome, error)) diff --git a/src/doltgres/doltgres/c/scan.c b/src/doltgres/doltgres/c/scan.c index 836204d817e13..7804063bc2972 100644 --- a/src/doltgres/doltgres/c/scan.c +++ b/src/doltgres/doltgres/c/scan.c @@ -51,6 +51,7 @@ #include "doltgres/txn/change.h" #include "doltgres/txn/counted.h" #include "doltgres/txn/insert.h" +#include "doltgres/txn/spill.h" #include "doltgres/txn/state.h" #include "doltgres/txn/written.h" #endif @@ -162,12 +163,16 @@ typedef enum DoltgresScanTurnTag DOLTGRES_SCAN_TURN_END, DOLTGRES_SCAN_TURN_RUN, DOLTGRES_SCAN_TURN_COMPARE, + DOLTGRES_SCAN_TURN_MOVED, } DoltgresScanTurnTag; /* * DoltgresScanTurn is Turn: what a turn of a read in the scan's record * leaves it, a row, another turn, the end, a compared table's run to make - * by its order, or two candidates to compare by it. + * by its order, two candidates to compare by it, or an install's move of the + * storage's base during the turn's read of the delta, which changed nothing + * in the record: the scan moves to the newest base (reseated) and takes the + * turn again. */ typedef struct DoltgresScanTurn { @@ -271,8 +276,12 @@ typedef struct DoltgresScanUnit /* * DoltgresScanParallel is Parallel: a parallel scan's part of its record, * the base's count of entries, the count of units, the unit being read, a - * tid range unit's first and last tuple ids, and the delta's split keys (a - * Vec>). + * tid range unit's first and last tuple ids, the delta's split keys (a + * Vec>), the table tree of the base the leader shared, whose + * entries' ordinals every worker's units split whatever newer base a worker + * moved to (rebased), and the unit's low and high keys (bounds), which a + * worker on a newer base than the shared one finds its unit's place and stop + * in. */ typedef struct DoltgresScanParallel { @@ -284,6 +293,19 @@ typedef struct DoltgresScanParallel uint64 ranged_first; uint64 ranged_last; StdVec splits; + bool has_shared; + TreesStoreTree shared; + doltgres_OptionVecU8_t low; + doltgres_OptionVecU8_t high; + + /* + * kept is the delta's run in the table's order (a Vec>), read at + * the scan's generation kept_at, which each unit takes its keys from + * (bugs row 309). + */ + bool has_kept; + StdVec kept; + uint64 kept_at; } DoltgresScanParallel; /* @@ -301,6 +323,12 @@ typedef struct DoltgresScanPaired uint32 table; TreesKeyOrder order; StdVec transform; + + /* + * resolved is the memo of the fetch's scan, or its call, which the pair + * doesn't own. + */ + DoltgresScanResolved *resolved; } DoltgresScanPaired; /* DoltgresScanScan is Scan: a table scan's record, its fields as the Rust's, in its order. */ @@ -321,6 +349,14 @@ struct DoltgresScanScan TreesStoreTree tree; uint64 folded; DoltgresScanEpochs epochs; + + /* + * resolved is the memo of the base leaf the scan's fetches last resolved + * a row in. + */ + DoltgresScanResolved resolved; + /* generation is the base generation the scan's base was read at. */ + uint64 generation; uint64 blocks; StdVec keyed; uint32 table; @@ -419,6 +455,47 @@ doltgres_scan_Epochs_of(HostIoIo *io, TreesStoreNodeStore store, uint32 epoch, b return true; } +/* doltgres_scan_internal returns an internal error of the fork's with a literal message. */ +static HostError +doltgres_scan_internal(const char *message) +{ + StdString text; + + std_string_from_Checked(&text, message, strlen(message)); + return host_Error_fork((const uint8 *) "XX000", &text); +} + +/* + * doltgres_scan_Epochs_moved takes the epochs of a newer base a read moved to + * (rebased, Trees_newer), which it takes, keeping the reader's epoch and its + * layout's check; a newer base whose epoch or chain doesn't reach the + * reader's epoch is an internal error, as a row past it is. + */ +static bool +doltgres_scan_Epochs_moved(DoltgresScanEpochs *self, DoltgresScanEpochs *newer, HostError *error) +{ + size_t end = 0; + + if (self->has_end) + { + size_t reached = (size_t) self->base + self->end; + + if (reached < newer->base || reached - newer->base > newer->chain.len) + { + doltgres_scan_Epochs_drop(newer); + *error = doltgres_scan_internal("a newer base is past its reader's epoch"); + return false; + } + end = reached - newer->base; + } + newer->has_end = self->has_end; + newer->end = end; + newer->checked = self->checked; + doltgres_scan_Epochs_drop(self); + *self = *newer; + return true; +} + /* doltgres_scan_Epochs_based answers a base's epoch and chain tree, which of reads, epoch 0 and no chain for none. */ void doltgres_scan_Epochs_based(const TxnBaseBase *base, uint32 *result_0, bool *has_result_1, TreesStoreTree *result_1) @@ -711,6 +788,7 @@ doltgres_scan_Turn_drop(DoltgresScanTurn *self) break; case DOLTGRES_SCAN_TURN_AGAIN: case DOLTGRES_SCAN_TURN_END: + case DOLTGRES_SCAN_TURN_MOVED: break; } } @@ -788,6 +866,16 @@ doltgres_scan_Paired_drop(DoltgresScanPaired *self) std_vec_free_Checked(&self->transform, DOLTGRES_SCAN_BYTES); } +/* doltgres_scan_Resolved_drop drops a memo of a base leaf: its stamps and its kept cursor, when it holds them. */ +void +doltgres_scan_Resolved_drop(DoltgresScanResolved *self) +{ + trees_cursor_LeafStamps_drop(&self->stamps); + if (self->has_kept) + trees_cursor_KeptCursor_drop(&self->kept); + self->has_kept = false; +} + /* doltgres_scan_Scan_drop drops a scan's record, its fields in their order, its reader first. */ static void doltgres_scan_Scan_drop(void *record) @@ -803,12 +891,19 @@ doltgres_scan_Scan_drop(void *record) if (scan->has_written) std_vec_free_Checked(&scan->written, DOLTGRES_SCAN_U64); doltgres_scan_Epochs_drop(&scan->epochs); + doltgres_scan_Resolved_drop(&scan->resolved); std_vec_free_Checked(&scan->keyed, DOLTGRES_SCAN_BYTES); trees_KeyOrder_drop(&scan->order); doltgres_scan_option_drop(&scan->bound); doltgres_scan_option_drop(&scan->seek); if (scan->has_parallel) + { doltgres_scan_vecs_drop(&scan->parallel.splits); + doltgres_scan_option_drop(&scan->parallel.low); + doltgres_scan_option_drop(&scan->parallel.high); + if (scan->parallel.has_kept) + doltgres_scan_vecs_drop(&scan->parallel.kept); + } doltgres_scan_Page_drop(&scan->page); if (scan->has_sample) doltgres_scan_rows_drop(&scan->sample.rows); @@ -1331,6 +1426,8 @@ static bool doltgres_scan_split(HostIoIo *io, const StdVec *keys, const uint8 *f static bool doltgres_scan_pick(HostIoIo *io, StdVec *base_key, DoltgresScanCopies *base_copies, StdVec *delta_key, size_t delta_at, const TreesKeyOrder *order, DoltgresScanDirection way, DoltgresScanPicked *result, HostError *error); +static bool doltgres_scan_reseated(HostIoIo *io, const DoltgresScanStores *stores, const DoltgresScanUnder *under, + HostOwnedHandle handle, DoltgresScanDirection way, HostError *error); /* doltgres_scan_Repo_Stores_with is the repository's Stores::with: the call lent the repository's reader of nodes. */ static void @@ -1426,6 +1523,7 @@ typedef struct doltgres_scan_begin_frame_t TreesStoreNodeStore store; uint64 blocks; const TxnBaseBase *base; + uint64 generation; StdVec keyed; bool keyed_live; uint32 table; @@ -1483,8 +1581,9 @@ doltgres_scan_begin_body(void *frame_) scan.has_tree = base != NULL && base->has_root; if (scan.has_tree) scan.tree = base->root; - scan.folded = base == NULL ? 0 : base->folded; + scan.folded = base == NULL ? 0 : base->read; scan.epochs = frame->epochs; + scan.generation = frame->generation; scan.blocks = frame->blocks; scan.keyed = frame->keyed; scan.table = frame->table; @@ -1501,16 +1600,18 @@ doltgres_scan_begin_body(void *frame_) /* * doltgres_scan_scan_begin makes a scan of a storage of a count of blocks, - * whose base, or no base (NULL), is in a table of a keyed layout, an object - * id, and a tree's order, which it takes, and answers its handle; its first - * read gives the storage's first row its snapshot sees. + * whose base, or no base (NULL), read at a base generation or after it, is in + * a table of a keyed layout, an object id, and a tree's order, which it + * takes, and answers its handle; its first read gives the storage's first + * row its snapshot sees. */ bool doltgres_scan_scan_begin(HostIoIo *io, TreesStoreNodeStore store, uint64 blocks, const TxnBaseBase *base, - StdVec *keyed, uint32 table, TreesKeyOrder *order, HostOwnedHandle *result, - HostError *error) + uint64 generation, StdVec *keyed, uint32 table, TreesKeyOrder *order, + HostOwnedHandle *result, HostError *error) { - doltgres_scan_begin_frame_t frame = {.io = io, .store = store, .blocks = blocks, .base = base, .keyed = *keyed, + doltgres_scan_begin_frame_t frame = {.io = io, .store = store, .blocks = blocks, .base = base, + .generation = generation, .keyed = *keyed, .keyed_live = true, .table = table, .order = *order, .order_live = true, .result = result, .error = error}; std_landing(&frame, doltgres_scan_begin_body, doltgres_scan_begin_pad); @@ -1629,8 +1730,8 @@ doltgres_scan_gather_closure_2(void *captures, HostIoIo *io, TreesStoreNodeStore /* * doltgres_scan_advance_frame_t is advance's state across its pad: the - * group's key it moves from, the base's entry, the delta's hold, the key a - * seek left, and the delta's key and versions, each while live. + * base's entry, the delta's hold, and the delta's key and versions, each + * while live. */ typedef struct doltgres_scan_advance_frame_t { @@ -1639,11 +1740,9 @@ typedef struct doltgres_scan_advance_frame_t const DoltgresScanUnder *under; DoltgresScanScan *scan; DoltgresScanDirection way; - doltgres_OptionVecU8_t from; doltgres_scan_base_read_t entry; HostDeltaHold hold; bool hold_live; - doltgres_OptionVecU8_t at; bool has_delta; txn_TupleVecU8VecVersion_t delta; DoltgresScanTurn *result; @@ -1682,12 +1781,10 @@ doltgres_scan_advance_pad(void *frame_) doltgres_scan_advance_frame_t *frame = frame_; doltgres_scan_delta_drop(&frame->has_delta, &frame->delta); - doltgres_scan_option_drop(&frame->at); if (frame->hold_live) host_delta_Hold_drop(&frame->hold); frame->hold_live = false; doltgres_scan_entry_drop(&frame->entry); - doltgres_scan_option_drop(&frame->from); } /* doltgres_scan_advance_body finds the nearer of the base's next entry and the delta's next key and makes its group. */ @@ -1698,6 +1795,9 @@ doltgres_scan_advance_body(void *frame_) DoltgresScanScan *scan = frame->scan; DoltgresScanDirection way = frame->way; bool entries = scan->index && !scan->records; + uint8 record = TXN_ENTRY_RECORD; + const uint8 *from = NULL; + size_t from_len = 0; StdVec key; bool has_copies = false; DoltgresScanCopies copies; @@ -1705,64 +1805,63 @@ doltgres_scan_advance_body(void *frame_) if (scan->has_group) { - frame->from.has = true; - frame->from.v = scan->group.key; - std_vec_free_Checked(&scan->group.base, DOLTGRES_SCAN_BYTES); - std_ptr_drop_in_place_slice(scan->group.rows.ptr, scan->group.rows.len, sizeof(DoltgresScanFound), - doltgres_scan_Found_drop); - std_vec_free_Checked(&scan->group.rows, DOLTGRES_SCAN_FOUND); - scan->has_group = false; + from = scan->group.key.ptr; + from_len = scan->group.key.len; } - frame->entry.scan = scan; - frame->entry.keyless = scan->keyed.len == 0; - frame->entry.entries = entries ? &scan->order : NULL; - frame->entry.way = way; - frame->entry.error = frame->error; - frame->stores->ops->with(frame->stores->self, frame->io, &scan->reader, doltgres_scan_advance_closure_1, - &frame->entry); - if (!frame->entry.ok) - return; - if (!frame->from.has && entries && way == DOLTGRES_SCAN_DIRECTION_BACKWARD) + else if (entries && way == DOLTGRES_SCAN_DIRECTION_BACKWARD) { - uint8 record = TXN_ENTRY_RECORD; - - std_slice_to_vec_Checked(&frame->from.v, &record, 1, DOLTGRES_SCAN_BYTES); - frame->from.has = true; + from = &record; + from_len = 1; } if (!host_delta_hold(frame->io, &frame->hold, frame->error)) return; frame->hold_live = true; if (scan->seek.has) { - frame->at = scan->seek; - scan->seek.has = false; - if (!txn_visible_versions(frame->io, &frame->hold, frame->under->state, frame->at.v.ptr, frame->at.v.len, + if (!txn_visible_versions(frame->io, &frame->hold, frame->under->state, scan->seek.v.ptr, scan->seek.v.len, scan->folded, &versions, frame->error)) return; if (versions.len == 0) { txn_memtable_version_versions_drop(&versions, 0); - if (!txn_visible_next(frame->io, &frame->hold, frame->under->state, true, frame->at.v.ptr, - frame->at.v.len, doltgres_scan_toward(way), scan->folded, &frame->has_delta, + if (!txn_visible_next(frame->io, &frame->hold, frame->under->state, true, scan->seek.v.ptr, + scan->seek.v.len, doltgres_scan_toward(way), scan->folded, &frame->has_delta, &frame->delta, frame->error)) return; - doltgres_scan_option_drop(&frame->at); } else { - frame->delta.f0 = frame->at.v; + std_vec_clone(&frame->delta.f0, &scan->seek.v, DOLTGRES_SCAN_BYTES); frame->delta.f1 = versions; - frame->at.has = false; frame->has_delta = true; } } - else if (!txn_visible_next(frame->io, &frame->hold, frame->under->state, frame->from.has, frame->from.v.ptr, - frame->from.has ? frame->from.v.len : 0, doltgres_scan_toward(way), scan->folded, - &frame->has_delta, &frame->delta, frame->error)) + else if (!txn_visible_next(frame->io, &frame->hold, frame->under->state, from != NULL, from, from_len, + doltgres_scan_toward(way), scan->folded, &frame->has_delta, &frame->delta, + frame->error)) return; if (frame->has_delta && entries && frame->delta.f0.len > 0 && ((const uint8 *) frame->delta.f0.ptr)[0] == TXN_ENTRY_RECORD) doltgres_scan_delta_drop(&frame->has_delta, &frame->delta); + if (txn_state_moved_past(frame->under->state, scan->generation)) + { + frame->result->tag = DOLTGRES_SCAN_TURN_MOVED; + frame->ok = true; + return; + } + if (scan->has_group) + doltgres_scan_Group_drop(&scan->group); + scan->has_group = false; + doltgres_scan_option_drop(&scan->seek); + frame->entry.scan = scan; + frame->entry.keyless = scan->keyed.len == 0; + frame->entry.entries = entries ? &scan->order : NULL; + frame->entry.way = way; + frame->entry.error = frame->error; + frame->stores->ops->with(frame->stores->self, frame->io, &scan->reader, doltgres_scan_advance_closure_1, + &frame->entry); + if (!frame->entry.ok) + return; if (!frame->entry.has_entry && !frame->has_delta) { DoltgresScanPlace start = {.tag = DOLTGRES_SCAN_PLACE_START}; @@ -1817,7 +1916,10 @@ doltgres_scan_advance_body(void *frame_) * the way a read goes and makes that key's group: the nearer of the base's * next entry and the delta's next key with a version past the group's key, * by their bytes, one group when they are the same key. With neither, the - * scan starts over. + * scan starts over. It reads the delta first and, when an install moved the + * storage's base since the scan took its own (txn_state_moved_past), + * changes nothing and answers Moved; so the delta it pairs with its base is + * the delta before any newer base's drop. */ static bool doltgres_scan_advance(HostIoIo *io, const DoltgresScanStores *stores, const DoltgresScanUnder *under, @@ -1832,8 +1934,9 @@ doltgres_scan_advance(HostIoIo *io, const DoltgresScanStores *stores, const Dolt } /* - * doltgres_scan_chosen_frame_t is chosen's state across its pad: the choice, - * the delta's hold, and the key's versions, while live. + * doltgres_scan_chosen_frame_t is chosen's state across its pad: the + * choice, with where the run's split goes, the delta's hold, and the key's + * versions, while live, and whether an install moved the base. */ typedef struct doltgres_scan_chosen_frame_t { @@ -1841,6 +1944,8 @@ typedef struct doltgres_scan_chosen_frame_t const DoltgresScanUnder *under; DoltgresScanScan *scan; DoltgresScanDirection way; + bool has_at; + size_t at; bool delta_key; StdVec key; bool key_live; @@ -1848,6 +1953,7 @@ typedef struct doltgres_scan_chosen_frame_t DoltgresScanCopies copies; HostDeltaHold hold; bool hold_live; + bool *moved; HostError *error; bool ok; } doltgres_scan_chosen_frame_t; @@ -1871,13 +1977,15 @@ doltgres_scan_chosen_pad(void *frame_) /* * doltgres_scan_chosen_body reads the delta's versions of the key, when the - * group takes the delta's key, and makes the group. + * group takes the delta's key, and, unless an install moved the base, moves + * the run's split and makes the group. */ static void doltgres_scan_chosen_body(void *frame_) { doltgres_scan_chosen_frame_t *frame = frame_; DoltgresScanScan *scan = frame->scan; + DoltgresScanDirection way = frame->way; StdVec versions; if (!host_delta_hold(frame->io, &frame->hold, frame->error)) @@ -1891,6 +1999,26 @@ doltgres_scan_chosen_body(void *frame_) } else std_vec_new(&versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + *frame->moved = txn_state_moved_past(frame->under->state, scan->generation); + if (*frame->moved) + { + txn_memtable_version_versions_drop(&versions, 0); + frame->ok = true; + return; + } + doltgres_scan_option_drop(&scan->seek); + if (scan->has_run) + { + if (frame->has_at && (frame->delta_key || way == DOLTGRES_SCAN_DIRECTION_BACKWARD)) + scan->run.split = frame->at + 1; + else if (frame->has_at) + scan->run.split = frame->at; + else if (way == DOLTGRES_SCAN_DIRECTION_FORWARD) + scan->run.split = scan->run.keys.len; + else + scan->run.split = 0; + } + doltgres_scan_took(&scan->place, frame->has_copies); frame->key_live = false; if (!doltgres_scan_grouped(frame->io, frame->under, &scan->epochs, &frame->key, frame->has_copies, &frame->copies, &versions, &scan->group, frame->error)) @@ -1907,17 +2035,17 @@ doltgres_scan_chosen_body(void *frame_) * doltgres_scan_chosen makes a compared table's next group from its * candidates as chosen, which it takes, the delta's key's versions read * again by the key, and moves the run's split past the keys at or before the - * group's. + * group's. It reads the delta key's versions first and, when an install + * moved the storage's base since the scan took its own + * (txn_state_moved_past), changes nothing and answers moved, as advance + * answers Moved. */ static bool doltgres_scan_chosen(HostIoIo *io, const DoltgresScanUnder *under, DoltgresScanScan *scan, - DoltgresScanPicked *picked, DoltgresScanDirection way, HostError *error) + DoltgresScanPicked *picked, DoltgresScanDirection way, bool *moved, HostError *error) { doltgres_scan_chosen_frame_t frame = {.io = io, .under = under, .scan = scan, .way = way, .key_live = true, - .error = error}; - bool has_at; - size_t at = 0; - bool delta_key; + .moved = moved, .error = error}; switch (picked->tag) { @@ -1925,15 +2053,14 @@ doltgres_scan_chosen(HostIoIo *io, const DoltgresScanUnder *under, DoltgresScanS frame.key = picked->base_key; frame.copies = picked->base_copies; frame.has_copies = true; - has_at = picked->has_before; - at = picked->before; - delta_key = false; + frame.has_at = picked->has_before; + frame.at = picked->before; break; case DOLTGRES_SCAN_PICKED_DELTA: frame.key = picked->delta_key; - has_at = true; - at = picked->delta_at; - delta_key = true; + frame.has_at = true; + frame.at = picked->delta_at; + frame.delta_key = true; break; case DOLTGRES_SCAN_PICKED_BOTH: default: @@ -1941,24 +2068,11 @@ doltgres_scan_chosen(HostIoIo *io, const DoltgresScanUnder *under, DoltgresScanS frame.key = picked->delta_key; frame.copies = picked->base_copies; frame.has_copies = true; - has_at = true; - at = picked->delta_at; - delta_key = true; + frame.has_at = true; + frame.at = picked->delta_at; + frame.delta_key = true; break; } - if (scan->has_run) - { - if (has_at && (delta_key || way == DOLTGRES_SCAN_DIRECTION_BACKWARD)) - scan->run.split = at + 1; - else if (has_at) - scan->run.split = at; - else if (way == DOLTGRES_SCAN_DIRECTION_FORWARD) - scan->run.split = scan->run.keys.len; - else - scan->run.split = 0; - } - doltgres_scan_took(&scan->place, frame.has_copies); - frame.delta_key = delta_key; std_landing(&frame, doltgres_scan_chosen_body, doltgres_scan_chosen_pad); doltgres_scan_chosen_pad(&frame); return frame.ok; @@ -2011,9 +2125,8 @@ doltgres_scan_gather_body(void *frame_) bool has_at; size_t at = 0; DoltgresScanPicked picked = {0}; + bool moved; - if (seeked) - doltgres_scan_option_drop(&scan->seek); if (!scan->has_group && seeked && way == DOLTGRES_SCAN_DIRECTION_FORWARD) { has_at = run->split < run->keys.len; @@ -2075,6 +2188,7 @@ doltgres_scan_gather_body(void *frame_) doltgres_scan_Group_drop(&scan->group); scan->has_group = false; doltgres_scan_Place_set(&scan->place, start); + doltgres_scan_option_drop(&scan->seek); frame->result->tag = DOLTGRES_SCAN_TURN_END; frame->ok = true; return; @@ -2104,9 +2218,9 @@ doltgres_scan_gather_body(void *frame_) picked.delta_at = frame->delta_at; } frame->entry.has_entry = frame->has_delta = false; - if (!doltgres_scan_chosen(frame->io, frame->under, scan, &picked, way, frame->error)) + if (!doltgres_scan_chosen(frame->io, frame->under, scan, &picked, way, &moved, frame->error)) return; - frame->result->tag = DOLTGRES_SCAN_TURN_AGAIN; + frame->result->tag = moved ? DOLTGRES_SCAN_TURN_MOVED : DOLTGRES_SCAN_TURN_AGAIN; frame->ok = true; } @@ -2182,6 +2296,15 @@ typedef struct doltgres_scan_TupleOptionVecU8VecVecU8_t #define HOST_OWNED_WITH_T_DROP(place) DOLTGRES_SCAN_SEEK_KEYS_DROP(place) #include "doltgres/host/owned_template.h" +/* doltgres_scan_generation_closure_0 is a read's look in the record: the base generation the scan took its base at. */ +static bool +doltgres_scan_generation_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, uint64 *result, + HostError *error) +{ + *result = scan->generation; + return true; +} + /* doltgres_scan_turn_t is a read's turn's captures: the store's source, what the read is under, and its way. */ typedef struct doltgres_scan_turn_t { @@ -2259,13 +2382,15 @@ doltgres_scan_next_closure_2(void *captures, HostIoIo *io, DoltgresScanScan *sca /* * doltgres_scan_chose_t is the captures of a call that makes the chosen group - * in the record: the choice, the read's state, and its way. + * in the record: the choice, the read's state, its way, and whether an + * install moved the base. */ typedef struct doltgres_scan_chose_t { DoltgresScanPicked picked; const DoltgresScanUnder *under; DoltgresScanDirection way; + bool moved; } doltgres_scan_chose_t; /* doltgres_scan_chose_drop frees the choice of a call that never ran. */ @@ -2281,7 +2406,7 @@ doltgres_scan_next_closure_3(void *captures, HostIoIo *io, DoltgresScanScan *sca { doltgres_scan_chose_t *chose = captures; - return doltgres_scan_chosen(io, chose->under, scan, &chose->picked, chose->way, error); + return doltgres_scan_chosen(io, chose->under, scan, &chose->picked, chose->way, &chose->moved, error); } /* @@ -2347,15 +2472,35 @@ doltgres_scan_next_body(void *frame_) frame->ok = true; return; } - if (step.tag == DOLTGRES_SCAN_TURN_RUN) + if (step.tag == DOLTGRES_SCAN_TURN_MOVED) + { + if (!doltgres_scan_reseated(frame->io, frame->stores, frame->under, frame->handle, frame->way, + frame->error)) + return; + } + else if (step.tag == DOLTGRES_SCAN_TURN_RUN) { doltgres_scan_TupleOptionVecU8VecVecU8_t looked; doltgres_scan_run_set_t set = {0}; + uint64 generation; frame->order = step.u.run; frame->order_live = true; if (!doltgres_scan_run(frame->io, frame->under, &frame->order, &frame->keys, frame->error)) return; + frame->keys_live = true; + if (!host_owned_TableDoltgresScanScan_withU64(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_generation_closure_0, NULL, NULL, &generation, + frame->error)) + return; + if (txn_state_moved_past(frame->under->state, generation)) + { + doltgres_scan_next_pad(frame); + if (!doltgres_scan_reseated(frame->io, frame->stores, frame->under, frame->handle, frame->way, + frame->error)) + return; + continue; + } frame->keys_live = false; if (!host_owned_TableDoltgresScanScan_withTupleOptionVecU8VecVecU8(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, @@ -2393,6 +2538,10 @@ doltgres_scan_next_body(void *frame_) doltgres_scan_chose_drop, frame->error)) return; doltgres_scan_next_pad(frame); + if (chose.moved && + !doltgres_scan_reseated(frame->io, frame->stores, frame->under, frame->handle, frame->way, + frame->error)) + return; } } } @@ -2544,7 +2693,7 @@ doltgres_scan_run_body(void *frame_) * version, in byte order, and sorts them by the table's order with ties * broken by their bytes, so that keys its comparator finds equal make * adjacent groups; it runs with no record lent, since the comparator is user - * code. + * code. It pauses at doltgres-scan-run first, so a case counts its reads. */ static bool doltgres_scan_run(HostIoIo *io, const DoltgresScanUnder *under, const TreesKeyOrder *order, StdVec *result, @@ -2552,6 +2701,8 @@ doltgres_scan_run(HostIoIo *io, const DoltgresScanUnder *under, const TreesKeyOr { doltgres_scan_run_frame_t frame = {.io = io, .under = under, .order = order, .result = result, .error = error}; + if (!host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-scan-run", error)) + return false; std_landing(&frame, doltgres_scan_run_body, doltgres_scan_run_pad); doltgres_scan_run_pad(&frame); return frame.ok; @@ -3146,7 +3297,7 @@ doltgres_scan_grouped(HostIoIo *io, const DoltgresScanUnder *under, DoltgresScan /* * doltgres_scan_rescan_closure_0 is rescan's call: the scan's place, group, - * run, seek, page, sample, and unit cleared. + * run, seek, page, sample, unit, and kept run cleared. */ static bool doltgres_scan_rescan_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) @@ -3172,7 +3323,12 @@ doltgres_scan_rescan_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *s scan->has_written = false; scan->records = false; if (scan->has_parallel) + { scan->parallel.has_unit = scan->parallel.has_ranged = false; + if (scan->parallel.has_kept) + doltgres_scan_vecs_drop(&scan->parallel.kept); + scan->parallel.has_kept = false; + } return true; } @@ -3188,7 +3344,11 @@ doltgres_scan_scan_rescan(HostIoIo *io, HostOwnedHandle handle, HostError *error NULL, NULL, error); } -/* doltgres_scan_placed_captures_t is placed's arguments in a pin scope: the tree, the key, the way, and the order. */ +/* + * doltgres_scan_placed_captures_t is placed's arguments in a pin scope: the + * tree, the key, the way, the order, and whether the key is a group's, which + * a newer base's place takes when the tree holds it (reseated). + */ typedef struct doltgres_scan_placed_captures_t { TreesStoreTree tree; @@ -3196,6 +3356,7 @@ typedef struct doltgres_scan_placed_captures_t size_t from_len; DoltgresScanDirection way; const TreesKeyOrder *order; + bool grouped; } doltgres_scan_placed_captures_t; /* doltgres_scan_placed_frame_t is placed's state across its pad: its cursor, while live. */ @@ -3273,11 +3434,15 @@ doltgres_scan_placed_body(void *frame_) } if (trees_cursor_Cursor_is_valid(&frame->cursor)) { + StdBytes key = trees_cursor_Cursor_key(&frame->cursor); + bool taken = captures->grouped && + std_bytes_Ord_cmp(key.ptr, key.len, captures->from, captures->from_len) == STD_ORDERING_EQUAL; + frame->result->tag = DOLTGRES_SCAN_PLACE_AT; frame->cursor_live = false; trees_cursor_Cursor_keep(&frame->cursor, &frame->result->u.at.cursor); frame->result->u.at.way = captures->way; - frame->result->u.at.taken = false; + frame->result->u.at.taken = taken; } else *frame->result = doltgres_scan_off(captures->way); @@ -3287,7 +3452,9 @@ doltgres_scan_placed_body(void *frame_) /* * doltgres_scan_placed answers a base's place at a key in its tree's order: * its first entry not below the key going forward, or its last not above it - * going backward, or off its end that way. + * going backward, or off its end that way; at a group's key the tree holds, + * the place is taken, so the next advance steps past it whichever way it + * goes, as an advance leaves a nearer entry its group didn't take untaken. */ static bool doltgres_scan_placed(void *captures, HostIoIo *io, TreesPinPin *pin, DoltgresScanPlace *result, HostError *error) @@ -3325,116 +3492,511 @@ doltgres_scan_place_read(void *captures, HostIoIo *io, TreesStoreNodeStore store read->error); } -/* doltgres_scan_seek_closure_1_0 is seek's store call, the base's place at the key found under the scan's reader. */ -static void -doltgres_scan_seek_closure_1_0(void *captures, HostIoIo *io, TreesStoreNodeStore store) +/* doltgres_scan_Trees_of returns the trees of a base, or of none (NULL), read at a base generation or after it. */ +DoltgresScanTrees +doltgres_scan_Trees_of(const TxnBaseBase *base, uint64 generation) { - doltgres_scan_place_read(captures, io, store); + DoltgresScanTrees trees = {.generation = generation}; + + if (base == NULL) + return trees; + trees.has_root = base->has_root; + if (trees.has_root) + trees.root = base->root; + trees.has_tuple_ids = base->has_tuple_ids; + if (trees.has_tuple_ids) + trees.tuple_ids = base->tuple_ids; + trees.folded = base->read; + return trees; } /* - * doltgres_scan_seek_compared_closure_1 is seek_compared's store call, the - * base's place at the key found under its own reader. + * DoltgresScanNewest is what a read whose base an install moved reads of its + * storage's newest base: whether the generation moved, the base, none for a + * storage with no base, the generation it read it at, and its epochs, each + * while live. */ +typedef struct DoltgresScanNewest +{ + bool moved; + bool some; + TxnBaseBase base; + uint64 now; + DoltgresScanEpochs epochs; + bool epochs_live; +} DoltgresScanNewest; + +/* doltgres_scan_Newest_drop frees a newest base's read, its epochs and its base. */ static void -doltgres_scan_seek_compared_closure_1(void *captures, HostIoIo *io, TreesStoreNodeStore store) +doltgres_scan_Newest_drop(DoltgresScanNewest *self) { - doltgres_scan_place_read(captures, io, store); + if (self->epochs_live) + doltgres_scan_Epochs_drop(&self->epochs); + self->epochs_live = false; + if (self->some) + txn_base_Base_drop(&self->base); + self->some = false; } -/* doltgres_scan_OptionKeyOrder_t is an Option. */ -typedef struct doltgres_scan_OptionKeyOrder_t +/* + * doltgres_scan_newest reads a storage's newest base for a snapshot to read, + * as readable gives it, with the generation it read it at and its epochs, + * its chain read through a store, once an install moved the generation past + * one a read took its base at, and nothing while it hasn't, which costs one + * load of the generation. + */ +static bool +doltgres_scan_newest(HostIoIo *io, TreesStoreNodeStore store, const DoltgresScanUnder *under, uint64 generation, + DoltgresScanNewest *result, HostError *error) { - bool has; - TreesKeyOrder v; -} doltgres_scan_OptionKeyOrder_t; + uint32 epoch; + bool has_chain; + TreesStoreTree chain; -/* DOLTGRES_SCAN_OPTION_KEY_ORDER_DROP drops an Option a call left. */ -#define DOLTGRES_SCAN_OPTION_KEY_ORDER_DROP(place) \ - do { \ - if ((place)->has) \ - trees_KeyOrder_drop(&(place)->v); \ - } while (0) + if (!txn_state_readable_since(io, under->state, under->snapshot, generation, &result->moved, &result->some, + &result->base, &result->now, error)) + return false; + if (!result->moved) + return true; + doltgres_scan_Epochs_based(result->some ? &result->base : NULL, &epoch, &has_chain, &chain); + if (!doltgres_scan_Epochs_of(io, store, epoch, has_chain, chain, &result->epochs, error)) + return false; + result->epochs_live = true; + return true; +} -/* Scan's Table's with at Option, for seek. */ -#define HOST_OWNED_SCOPE static -#define HOST_OWNED_DECLARE -#define HOST_OWNED_DEFINE -#define HOST_OWNED_PREFIX doltgres_scan_Scan -#define HOST_OWNED_RS DoltgresScanScan -#define HOST_OWNED_R DoltgresScanScan -#define HOST_OWNED_R_DROP doltgres_scan_Scan_drop -#define HOST_OWNED_KIND 0 -#define HOST_OWNED_TABLE_FN doltgres_scan_Scan_Owned_table -#define HOST_OWNED_WITH_TS OptionKeyOrder -#define HOST_OWNED_WITH_T doltgres_scan_OptionKeyOrder_t -#define HOST_OWNED_WITH_T_DROP(place) DOLTGRES_SCAN_OPTION_KEY_ORDER_DROP(place) -#include "doltgres/host/owned_template.h" +/* doltgres_scan_Trees_newer_frame_t is Trees_newer's state across its pad: the newest base's read. */ +typedef struct doltgres_scan_Trees_newer_frame_t +{ + DoltgresScanTrees *self; + HostIoIo *io; + TreesStoreNodeStore store; + const DoltgresScanUnder *under; + DoltgresScanEpochs *epochs; + DoltgresScanNewest newest; + HostError *error; + bool ok; +} doltgres_scan_Trees_newer_frame_t; -/* doltgres_scan_TupleOptionTreeOptionRun_t is an (Option, Option): a scan's tree and its run, taken. */ -typedef struct doltgres_scan_TupleOptionTreeOptionRun_t +/* doltgres_scan_Trees_newer_pad frees the newest base's read. */ +static void +doltgres_scan_Trees_newer_pad(void *frame_) { - bool has_tree; - TreesStoreTree tree; - bool has_run; - DoltgresScanRun run; -} doltgres_scan_TupleOptionTreeOptionRun_t; + doltgres_scan_Newest_drop(&((doltgres_scan_Trees_newer_frame_t *) frame_)->newest); +} -/* DOLTGRES_SCAN_TREE_RUN_DROP drops the run a call left. */ -#define DOLTGRES_SCAN_TREE_RUN_DROP(place) \ - do { \ - if ((place)->has_run) \ - doltgres_scan_vecs_drop(&(place)->run.keys); \ - } while (0) +/* doltgres_scan_Trees_newer_body reads the newest base and moves the read's trees and epochs to it. */ +static void +doltgres_scan_Trees_newer_body(void *frame_) +{ + doltgres_scan_Trees_newer_frame_t *frame = frame_; + DoltgresScanNewest *newest = &frame->newest; -/* Scan's Table's with at (Option, Option), for seek_compared. */ -#define HOST_OWNED_SCOPE static -#define HOST_OWNED_DECLARE -#define HOST_OWNED_DEFINE -#define HOST_OWNED_PREFIX doltgres_scan_Scan -#define HOST_OWNED_RS DoltgresScanScan -#define HOST_OWNED_R DoltgresScanScan -#define HOST_OWNED_R_DROP doltgres_scan_Scan_drop -#define HOST_OWNED_KIND 0 -#define HOST_OWNED_TABLE_FN doltgres_scan_Scan_Owned_table -#define HOST_OWNED_WITH_TS TupleOptionTreeOptionRun -#define HOST_OWNED_WITH_T doltgres_scan_TupleOptionTreeOptionRun_t -#define HOST_OWNED_WITH_T_DROP(place) DOLTGRES_SCAN_TREE_RUN_DROP(place) -#include "doltgres/host/owned_template.h" + if (!doltgres_scan_newest(frame->io, frame->store, frame->under, frame->self->generation, newest, frame->error)) + return; + if (newest->moved) + { + newest->epochs_live = false; + if (!doltgres_scan_Epochs_moved(frame->epochs, &newest->epochs, frame->error)) + return; + *frame->self = doltgres_scan_Trees_of(newest->some ? &newest->base : NULL, newest->now); + } + frame->ok = true; +} -/* doltgres_scan_seek_closure_0 is seek's look in the record: the order cloned out of it when a comparator orders it. */ +/* + * doltgres_scan_Trees_newer moves the trees of a read whose base an install + * moved to its storage's newest base, as readable gives it for the read's + * snapshot, with the generation it read it at, and the read's epochs to + * that base's (Epochs_moved), leaving them as they are while the base + * hasn't moved. + */ static bool -doltgres_scan_seek_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, - doltgres_scan_OptionKeyOrder_t *result, HostError *error) +doltgres_scan_Trees_newer(DoltgresScanTrees *self, HostIoIo *io, TreesStoreNodeStore store, + const DoltgresScanUnder *under, DoltgresScanEpochs *epochs, HostError *error) { - result->has = !trees_KeyOrder_is_bytes(&scan->order); - if (result->has) - trees_KeyOrder_Clone_clone(&scan->order, &result->v); - return true; + doltgres_scan_Trees_newer_frame_t frame = {.self = self, .io = io, .store = store, .under = under, + .epochs = epochs, .error = error}; + + std_landing(&frame, doltgres_scan_Trees_newer_body, doltgres_scan_Trees_newer_pad); + doltgres_scan_Trees_newer_pad(&frame); + return frame.ok; } -/* doltgres_scan_seek_t is a seek's captures: the store's source, the key, and the way. */ -typedef struct doltgres_scan_seek_t +/* + * DoltgresScanRebased is Rebased: what a move of a scan to its storage's + * newest base leaves its place to go on from: whether it moved, the new + * base's tree, the key a read goes on from, with whether it is its group's + * key, gone past, or a seek's, gone to, none for a scan not begun, and the + * table's order, cloned for the comparisons outside the record, while live. + */ +typedef struct DoltgresScanRebased { - const DoltgresScanStores *stores; - const uint8 *from; - size_t from_len; - DoltgresScanDirection way; -} doltgres_scan_seek_t; + bool moved; + bool has_tree; + TreesStoreTree tree; + doltgres_OptionVecU8_t from; + bool grouped; + TreesKeyOrder order; + bool order_live; +} DoltgresScanRebased; -/* doltgres_scan_seek_records returns whether a seek's key is an index's record. */ -static bool -doltgres_scan_seek_records(const uint8 *from, size_t from_len) +/* doltgres_scan_Rebased_drop frees a move's key and order, each while live. */ +static void +doltgres_scan_Rebased_drop(DoltgresScanRebased *self) { - return from_len > 0 && from[0] == TXN_ENTRY_RECORD; + doltgres_scan_option_drop(&self->from); + if (self->order_live) + trees_KeyOrder_drop(&self->order); + self->order_live = self->moved = false; } -/* doltgres_scan_seek_closure_1 is seek's placing in the record of a byte-ordered tree. */ -static bool -doltgres_scan_seek_closure_1(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +/* doltgres_scan_rebased_put_t is the captures of rebased's put in the record: the newest base's read and the move. */ +typedef struct doltgres_scan_rebased_put_t { - doltgres_scan_seek_t *seek = captures; - DoltgresScanPlace place = doltgres_scan_off(seek->way); + DoltgresScanNewest *newest; + DoltgresScanRebased *result; +} doltgres_scan_rebased_put_t; + +/* + * doltgres_scan_rebased_closure_0 is rebased's put in the record: the newest + * base's trees, read point, epochs, and generation, the tid range read's + * delta tuple ids cleared, and the key the scan goes on from. + */ +static bool +doltgres_scan_rebased_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + doltgres_scan_rebased_put_t *put = captures; + DoltgresScanNewest *newest = put->newest; + DoltgresScanRebased *result = put->result; + DoltgresScanTrees trees = doltgres_scan_Trees_of(newest->some ? &newest->base : NULL, newest->now); + + newest->epochs_live = false; + if (!doltgres_scan_Epochs_moved(&scan->epochs, &newest->epochs, error)) + return false; + scan->has_tree = trees.has_root; + scan->tree = trees.root; + scan->has_tuple_ids = trees.has_tuple_ids; + scan->tuple_ids = trees.tuple_ids; + scan->folded = trees.folded; + scan->generation = trees.generation; + if (scan->has_written) + std_vec_free_Checked(&scan->written, DOLTGRES_SCAN_U64); + scan->has_written = false; + result->moved = true; + result->has_tree = scan->has_tree; + result->tree = scan->tree; + result->grouped = scan->has_group; + if (scan->has_group) + std_vec_clone(&result->from.v, &scan->group.key, DOLTGRES_SCAN_BYTES); + else if (scan->seek.has) + std_vec_clone(&result->from.v, &scan->seek.v, DOLTGRES_SCAN_BYTES); + result->from.has = scan->has_group || scan->seek.has; + trees_KeyOrder_Clone_clone(&scan->order, &result->order); + result->order_live = true; + return true; +} + +/* doltgres_scan_rebased_frame_t is rebased's state across its pad: the newest base's read. */ +typedef struct doltgres_scan_rebased_frame_t +{ + HostIoIo *io; + TreesStoreNodeStore store; + const DoltgresScanUnder *under; + HostOwnedHandle handle; + DoltgresScanNewest newest; + DoltgresScanRebased *result; + HostError *error; + bool ok; +} doltgres_scan_rebased_frame_t; + +/* doltgres_scan_rebased_pad frees the newest base's read. */ +static void +doltgres_scan_rebased_pad(void *frame_) +{ + doltgres_scan_Newest_drop(&((doltgres_scan_rebased_frame_t *) frame_)->newest); +} + +/* doltgres_scan_rebased_body reads the newest base once the generation moved and puts it in the record. */ +static void +doltgres_scan_rebased_body(void *frame_) +{ + doltgres_scan_rebased_frame_t *frame = frame_; + doltgres_scan_rebased_put_t put = {&frame->newest, frame->result}; + uint64 generation; + + *frame->result = (DoltgresScanRebased) + { + .moved = false + }; + if (!host_owned_TableDoltgresScanScan_withU64(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_generation_closure_0, NULL, NULL, &generation, + frame->error) || + !doltgres_scan_newest(frame->io, frame->store, frame->under, generation, &frame->newest, frame->error)) + return; + frame->ok = !frame->newest.moved || + host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_rebased_closure_0, &put, NULL, frame->error); +} + +/* + * doltgres_scan_rebased moves a scan to its storage's newest base when an + * install moved the base since the scan took its own (bugs rows 240 and + * 241), at a read's start and never inside one: the record takes the new + * base's trees, read point, epochs (Epochs_moved), and generation, the + * epochs' chain read through a store, and a tid range read's delta tuple ids + * are collected again at that point by its next read. Every live snapshot + * may read the newest base (txn_state_readable), and its read point passes + * over the delta's versions it holds, so a read going on from its group's + * key reads nothing twice and misses nothing. It answers what the caller's + * place goes on from, or that the base hasn't moved, which costs one load of + * the generation. + */ +static bool +doltgres_scan_rebased(HostIoIo *io, TreesStoreNodeStore store, const DoltgresScanUnder *under, + HostOwnedHandle handle, DoltgresScanRebased *result, HostError *error) +{ + doltgres_scan_rebased_frame_t frame = {.io = io, .store = store, .under = under, .handle = handle, + .result = result, .error = error}; + + std_landing(&frame, doltgres_scan_rebased_body, doltgres_scan_rebased_pad); + doltgres_scan_rebased_pad(&frame); + return frame.ok; +} + +/* + * doltgres_scan_rebase_read_t is a move to the newest base under a reader, a + * Stores::with call's captures: rebased's arguments and its answer. + */ +typedef struct doltgres_scan_rebase_read_t +{ + const DoltgresScanUnder *under; + HostOwnedHandle handle; + DoltgresScanRebased *result; + HostError *error; + bool ok; +} doltgres_scan_rebase_read_t; + +/* doltgres_scan_rebase_read moves the scan to the newest base, reseated's and next_parallel's store calls' body. */ +static void +doltgres_scan_rebase_read(void *captures, HostIoIo *io, TreesStoreNodeStore store) +{ + doltgres_scan_rebase_read_t *read = captures; + + read->ok = doltgres_scan_rebased(io, store, read->under, read->handle, read->result, read->error); +} + +/* doltgres_scan_reseated_closure_0 is reseated's put of the place in the record. */ +static bool +doltgres_scan_reseated_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + doltgres_scan_Place_set(&scan->place, *(DoltgresScanPlace *) captures); + return true; +} + +/* doltgres_scan_place_put_drop drops the place of a put that never ran. */ +static void +doltgres_scan_place_put_drop(void *captures) +{ + doltgres_scan_Place_drop(captures); +} + +/* doltgres_scan_reseated_frame_t is reseated's state across its pad: its reader and the move, while live. */ +typedef struct doltgres_scan_reseated_frame_t +{ + HostIoIo *io; + const DoltgresScanStores *stores; + const DoltgresScanUnder *under; + HostOwnedHandle handle; + DoltgresScanDirection way; + HostSharedReader reading; + bool reading_live; + DoltgresScanRebased rebasing; + DoltgresScanPlace place; + HostError *error; + bool ok; +} doltgres_scan_reseated_frame_t; + +/* doltgres_scan_reseated_pad drops what reseated holds. */ +static void +doltgres_scan_reseated_pad(void *frame_) +{ + doltgres_scan_reseated_frame_t *frame = frame_; + + doltgres_scan_Rebased_drop(&frame->rebasing); + if (frame->reading_live) + host_shared_Reader_drop(&frame->reading); + frame->reading_live = false; +} + +/* doltgres_scan_reseated_body moves the scan to the newest base and puts its place in the new base's tree. */ +static void +doltgres_scan_reseated_body(void *frame_) +{ + doltgres_scan_reseated_frame_t *frame = frame_; + DoltgresScanRebased *rebasing = &frame->rebasing; + doltgres_scan_rebase_read_t rebase = {frame->under, frame->handle, rebasing, frame->error}; + + if (!host_shared_read(frame->io, &frame->reading, frame->error)) + return; + frame->reading_live = true; + frame->stores->ops->with(frame->stores->self, frame->io, &frame->reading, doltgres_scan_rebase_read, &rebase); + if (!rebase.ok) + return; + if (!rebasing->moved) + { + frame->ok = true; + return; + } + if (!rebasing->from.has) + frame->place.tag = DOLTGRES_SCAN_PLACE_START; + else if (rebasing->has_tree) + { + doltgres_scan_place_read_t read = {{rebasing->tree, rebasing->from.v.ptr, rebasing->from.v.len, frame->way, + &rebasing->order, rebasing->grouped}, .error = frame->error}; + + frame->stores->ops->with(frame->stores->self, frame->io, &frame->reading, doltgres_scan_place_read, &read); + if (!read.ok) + return; + frame->place = read.place; + } + else + frame->place = doltgres_scan_off(frame->way); + frame->ok = host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_reseated_closure_0, &frame->place, + doltgres_scan_place_put_drop, frame->error); +} + +/* + * doltgres_scan_reseated moves a scan whose read found an install moved its + * storage's base (txn_state_moved_past) to the newest base (rebased) and + * puts its place in the new base's tree, outside the record, since a + * compared tree's order runs user code: past its group's key (placed's + * grouped place), at a seek's key, or, for a scan not begun, whose read may + * have stepped its place in the old tree, at the new tree's start. + */ +static bool +doltgres_scan_reseated(HostIoIo *io, const DoltgresScanStores *stores, const DoltgresScanUnder *under, + HostOwnedHandle handle, DoltgresScanDirection way, HostError *error) +{ + doltgres_scan_reseated_frame_t frame = {.io = io, .stores = stores, .under = under, .handle = handle, + .way = way, .error = error}; + + std_landing(&frame, doltgres_scan_reseated_body, doltgres_scan_reseated_pad); + doltgres_scan_reseated_pad(&frame); + return frame.ok; +} + +/* doltgres_scan_seek_closure_1_0 is seek's store call, the base's place at the key found under the scan's reader. */ +static void +doltgres_scan_seek_closure_1_0(void *captures, HostIoIo *io, TreesStoreNodeStore store) +{ + doltgres_scan_place_read(captures, io, store); +} + +/* + * doltgres_scan_seek_compared_closure_1 is seek_compared's store call, the + * base's place at the key found under its own reader. + */ +static void +doltgres_scan_seek_compared_closure_1(void *captures, HostIoIo *io, TreesStoreNodeStore store) +{ + doltgres_scan_place_read(captures, io, store); +} + +/* doltgres_scan_OptionKeyOrder_t is an Option. */ +typedef struct doltgres_scan_OptionKeyOrder_t +{ + bool has; + TreesKeyOrder v; +} doltgres_scan_OptionKeyOrder_t; + +/* DOLTGRES_SCAN_OPTION_KEY_ORDER_DROP drops an Option a call left. */ +#define DOLTGRES_SCAN_OPTION_KEY_ORDER_DROP(place) \ + do { \ + if ((place)->has) \ + trees_KeyOrder_drop(&(place)->v); \ + } while (0) + +/* Scan's Table's with at Option, for seek. */ +#define HOST_OWNED_SCOPE static +#define HOST_OWNED_DECLARE +#define HOST_OWNED_DEFINE +#define HOST_OWNED_PREFIX doltgres_scan_Scan +#define HOST_OWNED_RS DoltgresScanScan +#define HOST_OWNED_R DoltgresScanScan +#define HOST_OWNED_R_DROP doltgres_scan_Scan_drop +#define HOST_OWNED_KIND 0 +#define HOST_OWNED_TABLE_FN doltgres_scan_Scan_Owned_table +#define HOST_OWNED_WITH_TS OptionKeyOrder +#define HOST_OWNED_WITH_T doltgres_scan_OptionKeyOrder_t +#define HOST_OWNED_WITH_T_DROP(place) DOLTGRES_SCAN_OPTION_KEY_ORDER_DROP(place) +#include "doltgres/host/owned_template.h" + +/* doltgres_scan_TupleOptionTreeOptionRun_t is an (Option, Option): a scan's tree and its run, taken. */ +typedef struct doltgres_scan_TupleOptionTreeOptionRun_t +{ + bool has_tree; + TreesStoreTree tree; + bool has_run; + DoltgresScanRun run; +} doltgres_scan_TupleOptionTreeOptionRun_t; + +/* DOLTGRES_SCAN_TREE_RUN_DROP drops the run a call left. */ +#define DOLTGRES_SCAN_TREE_RUN_DROP(place) \ + do { \ + if ((place)->has_run) \ + doltgres_scan_vecs_drop(&(place)->run.keys); \ + } while (0) + +/* Scan's Table's with at (Option, Option), for seek_compared. */ +#define HOST_OWNED_SCOPE static +#define HOST_OWNED_DECLARE +#define HOST_OWNED_DEFINE +#define HOST_OWNED_PREFIX doltgres_scan_Scan +#define HOST_OWNED_RS DoltgresScanScan +#define HOST_OWNED_R DoltgresScanScan +#define HOST_OWNED_R_DROP doltgres_scan_Scan_drop +#define HOST_OWNED_KIND 0 +#define HOST_OWNED_TABLE_FN doltgres_scan_Scan_Owned_table +#define HOST_OWNED_WITH_TS TupleOptionTreeOptionRun +#define HOST_OWNED_WITH_T doltgres_scan_TupleOptionTreeOptionRun_t +#define HOST_OWNED_WITH_T_DROP(place) DOLTGRES_SCAN_TREE_RUN_DROP(place) +#include "doltgres/host/owned_template.h" + +/* doltgres_scan_seek_closure_0 is seek's look in the record: the order cloned out of it when a comparator orders it. */ +static bool +doltgres_scan_seek_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, + doltgres_scan_OptionKeyOrder_t *result, HostError *error) +{ + result->has = !trees_KeyOrder_is_bytes(&scan->order); + if (result->has) + trees_KeyOrder_Clone_clone(&scan->order, &result->v); + return true; +} + +/* doltgres_scan_seek_t is a seek's captures: the store's source, the key, and the way. */ +typedef struct doltgres_scan_seek_t +{ + const DoltgresScanStores *stores; + const uint8 *from; + size_t from_len; + DoltgresScanDirection way; +} doltgres_scan_seek_t; + +/* doltgres_scan_seek_records returns whether a seek's key is an index's record. */ +static bool +doltgres_scan_seek_records(const uint8 *from, size_t from_len) +{ + return from_len > 0 && from[0] == TXN_ENTRY_RECORD; +} + +/* doltgres_scan_seek_closure_1 is seek's placing in the record of a byte-ordered tree. */ +static bool +doltgres_scan_seek_closure_1(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + doltgres_scan_seek_t *seek = captures; + DoltgresScanPlace place = doltgres_scan_off(seek->way); if (scan->has_group) doltgres_scan_Group_drop(&scan->group); @@ -4082,6 +4644,44 @@ doltgres_scan_resolve_closure_0(uint64 tuple_id) return host_Error_fork((const uint8 *) "XX001", &text); } +/* + * doltgres_scan_resolved_seek puts the frame's cursor at the pair's key in + * its tree: in the memo's kept leaf when the memo kept a cursor sought from + * the same root and that leaf holds the key, else by a seek from the root, + * answering whether the tree holds the key. + */ +static bool +doltgres_scan_resolved_seek(doltgres_scan_cursor_frame_t *frame, DoltgresScanPaired *paired, bool *at) +{ + DoltgresScanResolved *resolved = paired->resolved; + ChunksErrorError failed; + + *at = false; + if (resolved->has_kept) + { + bool same = resolved->kept.path.len > 0 && + trees_store_Tree_PartialEq_eq(&STD_VEC_AT(&resolved->kept.path, TreesCursorFrame, 0).tree, &paired->root); + + resolved->has_kept = false; + if (!same) + trees_cursor_KeptCursor_drop(&resolved->kept); + else + { + if (!trees_cursor_KeptCursor_resume(&resolved->kept, frame->io, frame->pin, &frame->cursor, &failed)) + return doltgres_scan_storage_failed(&failed, frame->error); + frame->cursor_live = true; + if (!trees_cursor_Cursor_find_in_node(&frame->cursor, frame->io, paired->key.ptr, paired->key.len, + &paired->order, at, &failed)) + return doltgres_scan_storage_failed(&failed, frame->error); + if (*at) + return true; + frame->cursor_live = false; + trees_cursor_Cursor_drop(&frame->cursor); + } + } + return doltgres_scan_seek_exact(frame, &paired->root, paired->key.ptr, paired->key.len, &paired->order, at); +} + /* doltgres_scan_resolve_closure_1_body seeks the pair's key and makes the row of its copy that holds the tuple id. */ static void doltgres_scan_resolve_closure_1_body(void *frame_) @@ -4093,49 +4693,43 @@ doltgres_scan_resolve_closure_1_body(void *frame_) bool at; uint64 count; StdBytes tuple; - const ChunksTrailerStamp *stamp = NULL; + uint64 copies; + bool some; + ChunksTrailerStamp stamp; - if (!doltgres_scan_seek_exact(frame, &paired->root, paired->key.ptr, paired->key.len, &paired->order, &at)) + if (!doltgres_scan_resolved_seek(frame, paired, &at)) return; if (!at) { *frame->error = doltgres_scan_resolve_closure_0(paired->tuple_id); return; } - if (!trees_cursor_Cursor_stamps(&frame->cursor, frame->io, frame->pin, &frame->stamps, &failed)) + if (!trees_cursor_LeafStamps_stamp_of(&paired->resolved->stamps, frame->io, frame->pin, &frame->cursor, paired->tuple_id, + &copies, &some, &stamp, &frame->values, &failed)) { doltgres_scan_storage_failed(&failed, frame->error); return; } - frame->stamps_live = true; + frame->values_live = true; if (!doltgres_scan_copies(&frame->cursor, paired->keyed.len == 0, &count, &tuple, frame->error)) return; - if (frame->stamps.len != count) + if (copies != count) { *frame->error = doltgres_scan_uneven(); return; } - for (size_t i = 0; i < frame->stamps.len && stamp == NULL; i++) - if (STD_VEC_AT(&frame->stamps, ChunksTrailerStamp, i).tuple == paired->tuple_id) - stamp = &STD_VEC_AT(&frame->stamps, ChunksTrailerStamp, i); - if (stamp == NULL) + if (!some) { *frame->error = doltgres_scan_resolve_closure_0(paired->tuple_id); return; } - if (!trees_cursor_Cursor_values(&frame->cursor, frame->io, frame->pin, &frame->values, &failed)) - { - doltgres_scan_storage_failed(&failed, frame->error); - return; - } - frame->values_live = true; std_slice_to_vec_Checked(&frame->tuple, tuple.ptr, tuple.len, DOLTGRES_SCAN_BYTES); frame->tuple_live = true; frame->bytes_live = true; doltgres_scan_value_bytes(frame->values.ptr, frame->values.len, &frame->bytes); *row = (DoltgresScanRow) { - .table = paired->table, .stamp = *stamp, .tuple = frame->tuple, .values = frame->bytes + .table = paired->table, .stamp = stamp, .tuple = frame->tuple, .values = frame->bytes }; frame->tuple_live = frame->bytes_live = false; row->transform = paired->transform; @@ -4144,6 +4738,9 @@ doltgres_scan_resolve_closure_1_body(void *frame_) std_vec_new(&paired->transform, DOLTGRES_SCAN_BYTES); std_vec_new(&paired->keyed, DOLTGRES_SCAN_BYTES); std_vec_new(&paired->key, DOLTGRES_SCAN_BYTES); + frame->cursor_live = false; + trees_cursor_Cursor_keep(&frame->cursor, &paired->resolved->kept); + paired->resolved->has_kept = true; frame->ok = true; } @@ -4196,6 +4793,7 @@ typedef struct doltgres_scan_fetch_frame_t TreesStoreNodeStore store; const DoltgresScanUnder *under; DoltgresScanEpochs *epochs; + DoltgresScanResolved *resolved; DoltgresScanTrees trees; DoltgresScanLayout layout; bool layout_live; @@ -4279,29 +4877,46 @@ doltgres_scan_fetch_body(void *frame_) DoltgresScanPaired paired; *frame->has_result = false; - if (!txn_memtable_tuples_get(frame->io, under->state, tuple, &frame->written.has, &frame->written.v, - frame->error)) - return; - if (!frame->written.has && frame->trees.has_tuple_ids && - !doltgres_scan_pair(frame->io, frame->store, frame->trees.tuple_ids, frame->tuple_id, &frame->paired, - frame->error)) + if (!host_delta_hold(frame->io, &frame->hold, frame->error)) return; - key = frame->written.has ? &frame->written.v : frame->paired.has ? &frame->paired.v : NULL; + frame->hold_live = true; + for (;;) + { + if (!txn_memtable_tuples_get(frame->io, under->state, tuple, &frame->written.has, &frame->written.v, + frame->error)) + return; + if (!frame->written.has && frame->trees.has_tuple_ids && + !doltgres_scan_pair(frame->io, frame->store, frame->trees.tuple_ids, frame->tuple_id, &frame->paired, + frame->error)) + return; + key = frame->written.has ? &frame->written.v : frame->paired.has ? &frame->paired.v : NULL; + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-fetch-keyed", frame->error)) + return; + if (key != NULL) + { + if (!txn_memtable_slice_row_versions(frame->io, &frame->hold, under->state, key->ptr, key->len, tuple, + frame->trees.folded, &slice, frame->error)) + return; + frame->versions = slice.versions; + frame->versions_live = true; + std_vec_new(&slice.versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + txn_memtable_slice_Slice_drop(&slice); + } + if (!txn_state_moved_past(under->state, frame->trees.generation)) + break; + if (frame->versions_live) + txn_memtable_version_versions_drop(&frame->versions, 0); + frame->versions_live = false; + doltgres_scan_option_drop(&frame->paired); + doltgres_scan_option_drop(&frame->written); + if (!doltgres_scan_Trees_newer(&frame->trees, frame->io, frame->store, under, frame->epochs, frame->error)) + return; + } if (key == NULL) { frame->ok = true; return; } - if (!host_delta_hold(frame->io, &frame->hold, frame->error)) - return; - frame->hold_live = true; - if (!txn_memtable_slice_row_versions(frame->io, &frame->hold, under->state, key->ptr, key->len, tuple, - frame->trees.folded, &slice, frame->error)) - return; - frame->versions = slice.versions; - frame->versions_live = true; - std_vec_new(&slice.versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); - txn_memtable_slice_Slice_drop(&slice); if (!txn_visible_seen(frame->io, rule, frame->versions.ptr, frame->versions.len, under->has_serial ? &under->serial.read : NULL, &frame->seen, frame->error)) return; @@ -4353,7 +4968,7 @@ doltgres_scan_fetch_body(void *frame_) { .tuple_id = frame->tuple_id, .key = frame->paired.v, .has_root = frame->trees.has_root, .root = frame->trees.root, .keyed = frame->layout.keyed, .table = frame->layout.table, - .order = frame->layout.order, .transform = frame->transform + .order = frame->layout.order, .transform = frame->transform, .resolved = frame->resolved }; frame->paired.has = frame->layout_live = frame->transform_live = false; if (!doltgres_scan_resolve(frame->io, frame->store, &paired, &frame->row, frame->error)) @@ -4388,14 +5003,20 @@ doltgres_scan_fetch_body(void *frame_) * or the base's tuple id tree pairs with the id, then the key's row version * with the id among those the snapshot sees in the delta, or else the base's * copy with the id, unless its ender in the delta ended it for the snapshot. - * It takes the layout. + * Once it read the delta, a read whose base an install moved + * (txn_state_moved_past) reads again from the newest base, so it never pairs + * a base with the delta a newer base's drop changed; a case holds it inside + * its delta read, between its tuple id's lookup and its versions' read + * (doltgres-fetch-keyed). It takes the layout. */ bool doltgres_scan_fetch(HostIoIo *io, TreesStoreNodeStore store, const DoltgresScanUnder *under, - DoltgresScanEpochs *epochs, DoltgresScanTrees trees, DoltgresScanLayout *layout, uint64 tuple_id, - bool *has_result, DoltgresScanRow *result, HostError *error) + DoltgresScanEpochs *epochs, DoltgresScanResolved *resolved, DoltgresScanTrees trees, + DoltgresScanLayout *layout, uint64 tuple_id, bool *has_result, DoltgresScanRow *result, + HostError *error) { - doltgres_scan_fetch_frame_t frame = {.io = io, .store = store, .under = under, .epochs = epochs, .trees = trees, + doltgres_scan_fetch_frame_t frame = {.io = io, .store = store, .under = under, .epochs = epochs, + .resolved = resolved, .trees = trees, .layout = *layout, .layout_live = true, .tuple_id = tuple_id, .has_result = has_result, .result = result, .error = error}; @@ -4532,21 +5153,33 @@ doltgres_scan_fetch_key_body(void *frame_) StdVec key; bool has_copies; - if (frame->trees.has_root) + if (!host_delta_hold(frame->io, &frame->hold, frame->error)) + return; + frame->hold_live = true; + for (;;) { - doltgres_scan_fetch_key_captures_t captures = {frame->trees.root, frame->key, frame->key_len, - &frame->layout.order, frame->layout.keyed.len == 0}; + if (frame->trees.has_root) + { + doltgres_scan_fetch_key_captures_t captures = {frame->trees.root, frame->key, frame->key_len, + &frame->layout.order, frame->layout.keyed.len == 0}; - if (!trees_pin_pin_ResultOptionCopies(frame->io, frame->store, doltgres_scan_fetch_key_closure_0, &captures, - NULL, &frame->copied, frame->error)) + if (!trees_pin_pin_ResultOptionCopies(frame->io, frame->store, doltgres_scan_fetch_key_closure_0, + &captures, NULL, &frame->copied, frame->error)) + return; + } + if (!txn_visible_versions(frame->io, &frame->hold, frame->under->state, frame->key, frame->key_len, + frame->trees.folded, &versions, frame->error)) + return; + if (!txn_state_moved_past(frame->under->state, frame->trees.generation)) + break; + txn_memtable_version_versions_drop(&versions, 0); + if (frame->copied.has) + doltgres_scan_Copies_drop(&frame->copied.v); + frame->copied.has = false; + if (!doltgres_scan_Trees_newer(&frame->trees, frame->io, frame->store, frame->under, frame->epochs, + frame->error)) return; } - if (!host_delta_hold(frame->io, &frame->hold, frame->error)) - return; - frame->hold_live = true; - if (!txn_visible_versions(frame->io, &frame->hold, frame->under->state, frame->key, frame->key_len, - frame->trees.folded, &versions, frame->error)) - return; std_slice_to_vec_Checked(&key, frame->key, frame->key_len, DOLTGRES_SCAN_BYTES); has_copies = frame->copied.has; frame->copied.has = false; @@ -4572,8 +5205,9 @@ doltgres_scan_fetch_key_body(void *frame_) * doltgres_scan_fetch_key answers the rows a snapshot sees of a row key in a * storage, read through a store, in the order a scan's group gives them: the * copies of the base's entry of the key that the key's versions in the delta - * leave seen, then the delta's rows it sees, oldest first. It takes the - * layout. + * leave seen, then the delta's rows it sees, oldest first. Once it read the + * delta, a read whose base an install moved reads again from the newest + * base, as fetch does. It takes the layout. */ bool doltgres_scan_fetch_key(HostIoIo *io, TreesStoreNodeStore store, const DoltgresScanUnder *under, @@ -4594,9 +5228,7 @@ typedef struct doltgres_scan_keyed_frame_t { HostIoIo *io; TreesStoreNodeStore store; - HostSharedRecord state; - bool has_tuple_ids; - TreesStoreTree tuple_ids; + const HostSharedHeld *held; uint64 tuple_id; doltgres_OptionVecU8_t written; bool *has_result; @@ -4613,19 +5245,30 @@ doltgres_scan_keyed_pad(void *frame_) doltgres_scan_option_drop(&((doltgres_scan_keyed_frame_t *) frame_)->written); } -/* doltgres_scan_keyed_body reads the delta's and the base's keys of the tuple id. */ +/* doltgres_scan_keyed_body reads the delta's key of the tuple id, then the newest base's. */ static void doltgres_scan_keyed_body(void *frame_) { doltgres_scan_keyed_frame_t *frame = frame_; HostRow tuple = {frame->tuple_id}; doltgres_OptionVecU8_t paired = {0}; + bool some; + TxnBaseBase base; + bool has_tuple_ids; + TreesStoreTree tuple_ids; - if (!txn_memtable_tuples_get(frame->io, frame->state, tuple, &frame->written.has, &frame->written.v, - frame->error)) + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-tuple-keyed", frame->error) || + !txn_memtable_tuples_get(frame->io, host_shared_Held_record(frame->held), tuple, &frame->written.has, + &frame->written.v, frame->error) || + !txn_state_newest(frame->io, frame->held, &some, &base, frame->error)) return; - if (frame->has_tuple_ids && - !doltgres_scan_pair(frame->io, frame->store, frame->tuple_ids, frame->tuple_id, &paired, frame->error)) + has_tuple_ids = some && base.has_tuple_ids; + if (has_tuple_ids) + tuple_ids = base.tuple_ids; + if (some) + txn_base_Base_drop(&base); + if (has_tuple_ids && + !doltgres_scan_pair(frame->io, frame->store, tuple_ids, frame->tuple_id, &paired, frame->error)) return; *frame->result_1 = paired.has; *frame->has_result = frame->written.has || paired.has; @@ -4641,18 +5284,20 @@ doltgres_scan_keyed_body(void *frame_) } /* - * doltgres_scan_keyed answers the row key a tuple id names in a storage, - * the delta's tuple id table's, else the base's tuple id tree's, read - * through a store, with whether the base holds the row, or none for a tuple - * id no layer holds. + * doltgres_scan_keyed answers the row key a tuple id names in a held + * storage, the delta's tuple id table's, else the newest base's tuple id + * tree's, read through a store, with whether that base holds the row, or + * none for a tuple id no layer holds. It reads the base after the delta, so + * a row a fold's drop takes out of the delta after the delta's read is in + * the base it reads (bugs rows 240 and 315); it first pauses at + * doltgres-tuple-keyed. */ bool -doltgres_scan_keyed(HostIoIo *io, TreesStoreNodeStore store, HostSharedRecord state, bool has_tuple_ids, - TreesStoreTree tuple_ids, uint64 tuple_id, bool *has_result, StdVec *result_0, bool *result_1, - HostError *error) +doltgres_scan_keyed(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint64 tuple_id, + bool *has_result, StdVec *result_0, bool *result_1, HostError *error) { - doltgres_scan_keyed_frame_t frame = {io, store, state, has_tuple_ids, tuple_ids, tuple_id, {0}, has_result, - result_0, result_1, error, false}; + doltgres_scan_keyed_frame_t frame = {io, store, held, tuple_id, {0}, has_result, result_0, result_1, error, + false}; std_landing(&frame, doltgres_scan_keyed_body, doltgres_scan_keyed_pad); doltgres_scan_keyed_pad(&frame); @@ -5140,15 +5785,22 @@ doltgres_scan_delta_tid(const uint64 *written, size_t written_len, bool has_give /* * doltgres_scan_OptionChosen_t is next_in_range's choice in the record: a * tuple id, the trees and layout its fetch reads, and the epochs moved out - * of the record, or none. + * of the record, or none, with whether an install moved the base while it + * collected the delta's tuple ids, which the read moves to (rebased) before + * it chooses again. The tuple ids it chooses from are the base's and the + * delta's from before any newer base's drop, so none is missed; each fetch + * reads its row from the newest base (fetch), and a read whose fetch found a + * newer base moves to it once, so its later fetches start there. */ typedef struct doltgres_scan_OptionChosen_t { bool has; + bool moved; uint64 tuple_id; DoltgresScanTrees trees; DoltgresScanLayout layout; DoltgresScanEpochs epochs; + DoltgresScanResolved resolved; } doltgres_scan_OptionChosen_t; /* DOLTGRES_SCAN_CHOSEN_DROP drops a choice a call left. */ @@ -5158,6 +5810,7 @@ typedef struct doltgres_scan_OptionChosen_t { \ doltgres_scan_Layout_drop(&(place)->layout); \ doltgres_scan_Epochs_drop(&(place)->epochs); \ + doltgres_scan_Resolved_drop(&(place)->resolved); \ } \ } while (0) @@ -5200,13 +5853,19 @@ doltgres_scan_next_in_range_closure_0(void *captures, HostIoIo *io, DoltgresScan bool has_delta; uint64 delta = 0; - result->has = false; + result->has = result->moved = false; if (!has_range) return true; if (!scan->has_written) { if (!doltgres_scan_written(io, read->under, scan->folded, &scan->written, error)) return false; + if (txn_state_moved_past(read->under->state, scan->generation)) + { + std_vec_free_Checked(&scan->written, DOLTGRES_SCAN_U64); + result->moved = true; + return true; + } scan->has_written = true; } if (scan->has_tuple_ids && @@ -5227,34 +5886,24 @@ doltgres_scan_next_in_range_closure_0(void *captures, HostIoIo *io, DoltgresScan trees_KeyOrder_Clone_clone(&scan->order, &result->layout.order); result->trees = (DoltgresScanTrees) { - scan->has_tree, scan->tree, scan->has_tuple_ids, scan->tuple_ids, scan->folded + scan->has_tree, scan->tree, scan->has_tuple_ids, scan->tuple_ids, scan->folded, scan->generation }; result->tuple_id = scan->given; result->epochs = scan->epochs; doltgres_scan_Epochs_Default_default(&scan->epochs); + result->resolved = scan->resolved; + scan->resolved = (DoltgresScanResolved) + { + 0 + }; result->has = true; return true; } -/* doltgres_scan_epochs_put_drop drops the epochs of a put that never ran. */ -static void -doltgres_scan_epochs_put_drop(void *epochs) -{ - doltgres_scan_Epochs_drop(epochs); -} - -/* doltgres_scan_next_in_range_closure_1 is next_in_range's put of the epochs back in the record. */ -static bool -doltgres_scan_next_in_range_closure_1(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) -{ - doltgres_scan_Epochs_drop(&scan->epochs); - scan->epochs = *(DoltgresScanEpochs *) captures; - return true; -} - /* * doltgres_scan_next_in_range_frame_t is next_in_range's state across its pad: - * the choice's epochs and the fetched row, while live. + * the choice's epochs and memo of a base leaf's stamps, and the fetched row, + * while live. */ typedef struct doltgres_scan_next_in_range_frame_t { @@ -5264,6 +5913,7 @@ typedef struct doltgres_scan_next_in_range_frame_t HostOwnedHandle handle; DoltgresScanDirection way; DoltgresScanEpochs epochs; + DoltgresScanResolved resolved; bool epochs_live; bool has_row; DoltgresScanRow row; @@ -5273,6 +5923,29 @@ typedef struct doltgres_scan_next_in_range_frame_t bool ok; } doltgres_scan_next_in_range_frame_t; +/* doltgres_scan_epochs_put_drop drops the epochs and memo of a put that never ran. */ +static void +doltgres_scan_epochs_put_drop(void *frame_) +{ + doltgres_scan_next_in_range_frame_t *frame = frame_; + + doltgres_scan_Epochs_drop(&frame->epochs); + doltgres_scan_Resolved_drop(&frame->resolved); +} + +/* doltgres_scan_next_in_range_closure_1 is next_in_range's put of the epochs and memo back in the record. */ +static bool +doltgres_scan_next_in_range_closure_1(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + doltgres_scan_next_in_range_frame_t *frame = captures; + + doltgres_scan_Epochs_drop(&scan->epochs); + scan->epochs = frame->epochs; + doltgres_scan_Resolved_drop(&scan->resolved); + scan->resolved = frame->resolved; + return true; +} + /* doltgres_scan_next_in_range_pad drops what next_in_range holds. */ static void doltgres_scan_next_in_range_pad(void *frame_) @@ -5283,7 +5956,7 @@ doltgres_scan_next_in_range_pad(void *frame_) doltgres_scan_Row_drop(&frame->row); frame->has_row = false; if (frame->epochs_live) - doltgres_scan_Epochs_drop(&frame->epochs); + doltgres_scan_epochs_put_drop(frame); frame->epochs_live = false; } @@ -5297,11 +5970,19 @@ doltgres_scan_next_in_range_body(void *frame_) for (;;) { doltgres_scan_OptionChosen_t chosen; + DoltgresScanRebased rebasing; if (!host_owned_TableDoltgresScanScan_withOptionChosen(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, doltgres_scan_next_in_range_closure_0, &read, NULL, &chosen, frame->error)) return; + if (chosen.moved) + { + if (!doltgres_scan_rebased(frame->io, frame->store, frame->under, frame->handle, &rebasing, frame->error)) + return; + doltgres_scan_Rebased_drop(&rebasing); + continue; + } if (!chosen.has) { *frame->has_result = false; @@ -5309,15 +5990,22 @@ doltgres_scan_next_in_range_body(void *frame_) return; } frame->epochs = chosen.epochs; + frame->resolved = chosen.resolved; frame->epochs_live = true; - if (!doltgres_scan_fetch(frame->io, frame->store, frame->under, &frame->epochs, chosen.trees, &chosen.layout, - chosen.tuple_id, &frame->has_row, &frame->row, frame->error)) + if (!doltgres_scan_fetch(frame->io, frame->store, frame->under, &frame->epochs, &frame->resolved, chosen.trees, + &chosen.layout, chosen.tuple_id, &frame->has_row, &frame->row, frame->error)) return; frame->epochs_live = false; if (!host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, - doltgres_scan_next_in_range_closure_1, &frame->epochs, + doltgres_scan_next_in_range_closure_1, frame, doltgres_scan_epochs_put_drop, frame->error)) return; + if (txn_state_moved_past(frame->under->state, chosen.trees.generation)) + { + if (!doltgres_scan_rebased(frame->io, frame->store, frame->under, frame->handle, &rebasing, frame->error)) + return; + doltgres_scan_Rebased_drop(&rebasing); + } if (frame->has_row) { *frame->has_result = true; @@ -5415,16 +6103,6 @@ doltgres_scan_splits(const StdVec *starts, size_t starts_len, StdVec *result) std_landing(&frame, doltgres_scan_splits_body, doltgres_scan_splits_pad); } -/* doltgres_scan_internal returns an internal error of the fork's with a literal message. */ -static HostError -doltgres_scan_internal(const char *message) -{ - StdString text; - - std_string_from_Checked(&text, message, strlen(message)); - return host_Error_fork((const uint8 *) "XX000", &text); -} - /* doltgres_scan_split_keys_frame_t is split_keys' state across its pad: the keys read so far, and the bytes left. */ typedef struct doltgres_scan_split_keys_frame_t { @@ -5496,6 +6174,7 @@ typedef struct doltgres_scan_begin_parallel_frame_t HostIoIo *io; DoltgresScanShared shared; bool shared_live; + uint64 generation; StdVec keyed; bool keyed_live; uint32 table; @@ -5545,6 +6224,7 @@ doltgres_scan_begin_parallel_body(void *frame_) scan.tree = shared->tree; scan.folded = shared->folded; scan.epochs = shared->epochs; + scan.generation = frame->generation; scan.blocks = shared->has_tree ? (shared->fresh_end >> 16) + 1 : 0; scan.keyed = frame->keyed; scan.table = frame->table; @@ -5555,6 +6235,8 @@ doltgres_scan_begin_parallel_body(void *frame_) scan.parallel.entries = shared->entries; scan.parallel.units = shared->units; scan.parallel.splits = shared->splits; + scan.parallel.has_shared = shared->has_tree; + scan.parallel.shared = shared->tree; scan.page = doltgres_scan_ended(); frame->reader_live = frame->shared_live = frame->keyed_live = frame->order_live = false; frame->ok = host_owned_TableDoltgresScanScan_make(&DOLTGRES_SCAN_SCANS, frame->io, scan, frame->result, @@ -5563,15 +6245,16 @@ doltgres_scan_begin_parallel_body(void *frame_) /* * doltgres_scan_begin_parallel makes a parallel scan of the base the leader - * shared, in a table of a keyed layout, an object id, and a tree's order, - * each of which it takes, and answers its handle; its first read asks for - * its first unit. + * shared, the storage's newest at a base generation or none newer, in a + * table of a keyed layout, an object id, and a tree's order, each of which + * it takes, and answers its handle; its first read asks for its first unit. */ bool -doltgres_scan_begin_parallel(HostIoIo *io, DoltgresScanShared *shared, StdVec *keyed, uint32 table, - TreesKeyOrder *order, HostOwnedHandle *result, HostError *error) +doltgres_scan_begin_parallel(HostIoIo *io, DoltgresScanShared *shared, uint64 generation, StdVec *keyed, + uint32 table, TreesKeyOrder *order, HostOwnedHandle *result, HostError *error) { - doltgres_scan_begin_parallel_frame_t frame = {.io = io, .shared = *shared, .shared_live = true, .keyed = *keyed, + doltgres_scan_begin_parallel_frame_t frame = {.io = io, .shared = *shared, .shared_live = true, + .generation = generation, .keyed = *keyed, .keyed_live = true, .table = table, .order = *order, .order_live = true, .result = result, .error = error}; @@ -5784,6 +6467,20 @@ doltgres_scan_bounds_closure_1(const StdVec *splits, uint64 segment, uint64 segm return key; } +/* doltgres_scan_edges keeps a unit's bounds as its edges in the scan's parallel part, cloned. */ +static void +doltgres_scan_edges(DoltgresScanParallel *parallel, const DoltgresScanBounds *bounds) +{ + doltgres_scan_option_drop(¶llel->low); + doltgres_scan_option_drop(¶llel->high); + parallel->low.has = bounds->low.has; + if (bounds->low.has) + std_vec_clone(¶llel->low.v, &bounds->low.v, DOLTGRES_SCAN_BYTES); + parallel->high.has = bounds->high.has; + if (bounds->high.has) + std_vec_clone(¶llel->high.v, &bounds->high.v, DOLTGRES_SCAN_BYTES); +} + /* * doltgres_scan_bounds_frame_t is bounds' state across its pad: the scan, the * unit, the store's source, and the bounds being made. @@ -5821,7 +6518,7 @@ doltgres_scan_bounds_body(void *frame_) doltgres_scan_bounds_frame_t *frame = frame_; DoltgresScanScan *scan = frame->scan; DoltgresScanBounds *result = frame->result; - doltgres_scan_bounds_captures_t captures = {.scan = scan, .tree = scan->tree, .unit = frame->unit, + doltgres_scan_bounds_captures_t captures = {.scan = scan, .tree = scan->parallel.shared, .unit = frame->unit, .error = frame->error}; *result = (DoltgresScanBounds) @@ -5836,7 +6533,7 @@ doltgres_scan_bounds_body(void *frame_) return; } captures.entries = scan->parallel.entries; - if (!scan->has_tree || captures.entries == 0) + if (!scan->parallel.has_shared || captures.entries == 0) { const StdVec *splits = &scan->parallel.splits; uint64 segments = (uint64) splits->len + 1; @@ -5856,6 +6553,7 @@ doltgres_scan_bounds_body(void *frame_) std_vec_new(&result->high.v, DOLTGRES_SCAN_BYTES); } result->alone = !result->low.has && !result->high.has; + doltgres_scan_edges(&scan->parallel, result); frame->live = false; frame->ok = true; return; @@ -5870,6 +6568,7 @@ doltgres_scan_bounds_body(void *frame_) if (result->high.has) std_vec_clone(&scan->bound.v, &result->high.v, DOLTGRES_SCAN_BYTES); scan->bound.has = result->high.has; + doltgres_scan_edges(&scan->parallel, result); frame->live = false; frame->ok = true; } @@ -5893,25 +6592,77 @@ doltgres_scan_bounds(HostIoIo *io, const DoltgresScanStores *stores, DoltgresSca return frame.ok; } +/* + * doltgres_scan_Kept_t is a parallel worker's kept run outside its record: + * whether it has one, its keys (a Vec>), and the scan's generation + * it was read at, which a take sets to the record's generation when it finds + * none to give. + */ +typedef struct doltgres_scan_Kept_t +{ + bool has; + StdVec keys; + uint64 at; +} doltgres_scan_Kept_t; + +/* + * doltgres_scan_kept_take_closure_0 is kept_take's call: the record's kept run + * moved out when it was read at the scan's generation, and dropped when it + * wasn't, since a rebase or a rescan left it behind. + */ +static bool +doltgres_scan_kept_take_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + doltgres_scan_Kept_t *kept = captures; + + kept->has = false; + kept->at = scan->generation; + if (!scan->has_parallel || !scan->parallel.has_kept) + return true; + scan->parallel.has_kept = false; + if (scan->parallel.kept_at != scan->generation) + { + doltgres_scan_vecs_drop(&scan->parallel.kept); + return true; + } + kept->has = true; + kept->keys = scan->parallel.kept; + return true; +} + +/* doltgres_scan_kept_put_closure_0 is kept_put's call: the kept run moved back into the record. */ +static bool +doltgres_scan_kept_put_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + doltgres_scan_Kept_t *kept = captures; + + if (!scan->has_parallel) + return true; + scan->parallel.kept = kept->keys; + scan->parallel.kept_at = kept->at; + scan->parallel.has_kept = true; + kept->has = false; + return true; +} + /* * doltgres_scan_unit_keys_frame_t is unit_keys' state across its pad: the - * run's keys not yet looked at and the ones kept, while live. + * kept run while it is out of the record, and the keys kept, while live. */ typedef struct doltgres_scan_unit_keys_frame_t { HostIoIo *io; const DoltgresScanUnder *under; + HostOwnedHandle handle; const DoltgresScanBounds *bounds; - StdVec keys; - size_t next; - bool keys_live; + doltgres_scan_Kept_t kept; StdVec *result; bool result_live; HostError *error; bool ok; } doltgres_scan_unit_keys_frame_t; -/* doltgres_scan_unit_keys_pad drops the keys not yet looked at and the ones kept. */ +/* doltgres_scan_unit_keys_pad drops the kept run while it is out of the record, and the keys kept. */ static void doltgres_scan_unit_keys_pad(void *frame_) { @@ -5920,21 +6671,51 @@ doltgres_scan_unit_keys_pad(void *frame_) if (frame->result_live) doltgres_scan_vecs_drop(frame->result); frame->result_live = false; - if (frame->keys_live) + if (frame->kept.has) + doltgres_scan_vecs_drop(&frame->kept.keys); + frame->kept.has = false; +} + +/* + * doltgres_scan_unit_keys_at returns the place in the kept run of its first + * key at or past a bound in the table's order, by a binary search, since the + * run is in that order. + */ +static bool +doltgres_scan_unit_keys_at(doltgres_scan_unit_keys_frame_t *frame, const StdVec *bound, size_t *result) +{ + size_t low = 0; + size_t high = frame->kept.keys.len; + + while (low < high) { - for (size_t i = frame->next; i < frame->keys.len; i++) - std_vec_free_Checked(&STD_VEC_AT(&frame->keys, StdVec, i), DOLTGRES_SCAN_BYTES); - std_vec_free_Checked(&frame->keys, DOLTGRES_SCAN_VEC); + size_t middle = low + (high - low) / 2; + StdOrdering ordering; + + if (!doltgres_scan_total(frame->io, &frame->bounds->order, &STD_VEC_AT(&frame->kept.keys, StdVec, middle), + bound, &ordering, frame->error)) + return false; + if (ordering == STD_ORDERING_LESS) + low = middle + 1; + else + high = middle; } - frame->keys_live = false; + *result = low; + return true; } -/* doltgres_scan_unit_keys_body reads the run and keeps its keys within the unit's bounds. */ +/* + * doltgres_scan_unit_keys_body takes the kept run out of the record, reading + * the run when the record keeps none at the scan's generation, copies out its + * keys within the unit's bounds, and puts the run back. + */ static void doltgres_scan_unit_keys_body(void *frame_) { doltgres_scan_unit_keys_frame_t *frame = frame_; const DoltgresScanBounds *bounds = frame->bounds; + size_t first = 0; + size_t end; std_vec_new(frame->result, DOLTGRES_SCAN_VEC); if (!bounds->low.has && !bounds->high.has && !bounds->alone) @@ -5943,39 +6724,30 @@ doltgres_scan_unit_keys_body(void *frame_) return; } frame->result_live = true; - if (!doltgres_scan_run(frame->io, frame->under, &bounds->order, &frame->keys, frame->error)) + if (!host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_kept_take_closure_0, &frame->kept, NULL, frame->error)) return; - frame->keys_live = true; - while (frame->next < frame->keys.len) + if (!frame->kept.has) { - StdVec key = STD_VEC_AT(&frame->keys, StdVec, frame->next); - StdOrdering ordering; - bool at_or_past = true; - bool before = true; + if (!doltgres_scan_run(frame->io, frame->under, &bounds->order, &frame->kept.keys, frame->error)) + return; + frame->kept.has = true; + } + end = frame->kept.keys.len; + if ((bounds->low.has && !doltgres_scan_unit_keys_at(frame, &bounds->low.v, &first)) || + (bounds->high.has && !doltgres_scan_unit_keys_at(frame, &bounds->high.v, &end))) + return; + for (size_t i = first; i < end; i++) + { + const StdVec *key = &STD_VEC_AT(&frame->kept.keys, StdVec, i); + StdVec copy; - if (bounds->low.has) - { - if (!doltgres_scan_total(frame->io, &bounds->order, &key, &bounds->low.v, &ordering, frame->error)) - return; - at_or_past = ordering != STD_ORDERING_LESS; - } - if (bounds->high.has) - { - if (!doltgres_scan_total(frame->io, &bounds->order, &key, &bounds->high.v, &ordering, frame->error)) - return; - before = ordering == STD_ORDERING_LESS; - } - if (at_or_past && before) - { - std_vec_push_Checked(frame->result, &key, DOLTGRES_SCAN_VEC); - frame->next++; - } - else - { - frame->next++; - std_vec_free_Checked(&key, DOLTGRES_SCAN_BYTES); - } + std_slice_to_vec_Checked(©, key->ptr, key->len, DOLTGRES_SCAN_BYTES); + std_vec_push_Checked(frame->result, ©, DOLTGRES_SCAN_VEC); } + if (!host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_kept_put_closure_0, &frame->kept, NULL, frame->error)) + return; frame->result_live = false; frame->ok = true; } @@ -5984,14 +6756,16 @@ doltgres_scan_unit_keys_body(void *frame_) * doltgres_scan_unit_keys answers a parallel unit's delta keys in the * table's order: every key at or past its low bound and before its high * one, with ties broken by the keys' bytes, or every key when its bounds say - * the unit reads them all alone. + * the unit reads them all alone. It takes them from the run the worker keeps, + * read once for each generation the scan reads at, so a scan's units read the + * delta's run once and not once each (bugs row 309). */ static bool -doltgres_scan_unit_keys(HostIoIo *io, const DoltgresScanUnder *under, const DoltgresScanBounds *bounds, - StdVec *result, HostError *error) +doltgres_scan_unit_keys(HostIoIo *io, const DoltgresScanUnder *under, HostOwnedHandle handle, + const DoltgresScanBounds *bounds, StdVec *result, HostError *error) { - doltgres_scan_unit_keys_frame_t frame = {.io = io, .under = under, .bounds = bounds, .result = result, - .error = error}; + doltgres_scan_unit_keys_frame_t frame = {.io = io, .under = under, .handle = handle, .bounds = bounds, + .result = result, .error = error}; std_landing(&frame, doltgres_scan_unit_keys_body, doltgres_scan_unit_keys_pad); doltgres_scan_unit_keys_pad(&frame); @@ -6089,28 +6863,319 @@ doltgres_scan_next_parallel_closure_2(void *captures, HostIoIo *io, DoltgresScan static bool doltgres_scan_next_parallel_closure_3(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) { - if (scan->has_parallel) - scan->parallel.has_unit = false; - return true; + if (scan->has_parallel) + scan->parallel.has_unit = false; + return true; +} + +/* doltgres_scan_next_parallel_closure_4 is next_parallel's put of the delta's run in the record. */ +static bool +doltgres_scan_next_parallel_closure_4(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + doltgres_scan_run_put(scan, captures); + return true; +} + +/* doltgres_scan_next_parallel_closure_5 is next_parallel's making of the chosen group in the record, forward. */ +static bool +doltgres_scan_next_parallel_closure_5(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + doltgres_scan_chose_t *chose = captures; + + return doltgres_scan_chosen(io, chose->under, scan, &chose->picked, DOLTGRES_SCAN_DIRECTION_FORWARD, + &chose->moved, error); +} + +/* doltgres_scan_sought_captures_t is sought's pin scope's captures: the tree, the key, and its order. */ +typedef struct doltgres_scan_sought_captures_t +{ + TreesStoreTree tree; + const uint8 *key; + size_t key_len; + const TreesKeyOrder *order; +} doltgres_scan_sought_captures_t; + +/* doltgres_scan_sought_closure_0_body seeks the key and copies out the key of the first entry at or past it. */ +static void +doltgres_scan_sought_closure_0_body(void *frame_) +{ + doltgres_scan_cursor_frame_t *frame = frame_; + doltgres_scan_sought_captures_t *captures = frame->captures; + doltgres_OptionVecU8_t *result = frame->result; + ChunksErrorError failed; + StdBytes key; + + if (!trees_cursor_Cursor_seek(frame->io, frame->pin, TREES_STORE_TREE_NAMED(&captures->tree), captures->key, + captures->key_len, captures->order, &frame->cursor, &failed)) + { + doltgres_scan_storage_failed(&failed, frame->error); + return; + } + frame->cursor_live = true; + result->has = trees_cursor_Cursor_is_valid(&frame->cursor); + if (result->has) + { + key = trees_cursor_Cursor_key(&frame->cursor); + std_slice_to_vec_Checked(&result->v, key.ptr, key.len, DOLTGRES_SCAN_BYTES); + } + frame->ok = true; +} + +/* doltgres_scan_sought_closure_0 is sought's pin scope. */ +static bool +doltgres_scan_sought_closure_0(void *captures, HostIoIo *io, TreesPinPin *pin, doltgres_OptionVecU8_t *result, + HostError *error) +{ + return doltgres_scan_cursor_scope(captures, io, pin, result, error, doltgres_scan_sought_closure_0_body); +} + +/* doltgres_scan_sought_read_t is a stop's read under a reader, a Stores::with call's captures. */ +typedef struct doltgres_scan_sought_read_t +{ + doltgres_scan_sought_captures_t sought; + doltgres_OptionVecU8_t *result; + HostError *error; + bool ok; +} doltgres_scan_sought_read_t; + +/* doltgres_scan_sought_read reads the key of a tree's first entry at or past a key going forward, in a pin scope. */ +static void +doltgres_scan_sought_read(void *captures, HostIoIo *io, TreesStoreNodeStore store) +{ + doltgres_scan_sought_read_t *read = captures; + + read->ok = trees_pin_pin_ResultOptionVecU8(io, store, doltgres_scan_sought_closure_0, &read->sought, NULL, + read->result, read->error); +} + +/* + * DoltgresScanUnitPlacing is what unit_placed takes out of a parallel scan's + * record: whether the worker's tree is a newer base's than the shared one, + * that tree, its group's key, its unit's edges, and the table's order, each + * cloned and each while live, and the place and stop it puts back. + */ +typedef struct DoltgresScanUnitPlacing +{ + bool moved; + bool has_tree; + TreesStoreTree tree; + doltgres_OptionVecU8_t group; + doltgres_OptionVecU8_t low; + doltgres_OptionVecU8_t high; + TreesKeyOrder order; + bool order_live; + DoltgresScanPlace place; + doltgres_OptionVecU8_t bound; +} DoltgresScanUnitPlacing; + +/* doltgres_scan_unit_placed_closure_0 is unit_placed's look in the record: a moved worker's tree, group, and edges. */ +static bool +doltgres_scan_unit_placed_closure_0(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + DoltgresScanUnitPlacing *placing = captures; + const DoltgresScanParallel *parallel = &scan->parallel; + + if (!scan->has_parallel || !parallel->has_unit || + (scan->has_tree == parallel->has_shared && + (!scan->has_tree || trees_store_Tree_PartialEq_eq(&scan->tree, ¶llel->shared)))) + return true; + placing->moved = true; + placing->has_tree = scan->has_tree; + placing->tree = scan->tree; + if (scan->has_group) + std_vec_clone(&placing->group.v, &scan->group.key, DOLTGRES_SCAN_BYTES); + placing->group.has = scan->has_group; + if (parallel->low.has) + std_vec_clone(&placing->low.v, ¶llel->low.v, DOLTGRES_SCAN_BYTES); + placing->low.has = parallel->low.has; + if (parallel->high.has) + std_vec_clone(&placing->high.v, ¶llel->high.v, DOLTGRES_SCAN_BYTES); + placing->high.has = parallel->high.has; + trees_KeyOrder_Clone_clone(&scan->order, &placing->order); + placing->order_live = true; + return true; +} + +/* doltgres_scan_unit_placed_closure_1 is unit_placed's put of the place and the stop in the record. */ +static bool +doltgres_scan_unit_placed_closure_1(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +{ + DoltgresScanUnitPlacing *placing = captures; + + doltgres_scan_Place_set(&scan->place, placing->place); + doltgres_scan_option_drop(&scan->bound); + scan->bound = placing->bound; + placing->place.tag = DOLTGRES_SCAN_PLACE_START; + placing->bound.has = false; + return true; +} + +/* doltgres_scan_unit_placed_frame_t is unit_placed's state across its pad: its reader and what it took, while live. */ +typedef struct doltgres_scan_unit_placed_frame_t +{ + HostIoIo *io; + const DoltgresScanStores *stores; + HostOwnedHandle handle; + HostSharedReader reading; + bool reading_live; + DoltgresScanUnitPlacing placing; + HostError *error; + bool ok; +} doltgres_scan_unit_placed_frame_t; + +/* doltgres_scan_unit_placed_pad drops what unit_placed holds. */ +static void +doltgres_scan_unit_placed_pad(void *frame_) +{ + doltgres_scan_unit_placed_frame_t *frame = frame_; + DoltgresScanUnitPlacing *placing = &frame->placing; + + doltgres_scan_option_drop(&placing->bound); + doltgres_scan_Place_drop(&placing->place); + placing->place.tag = DOLTGRES_SCAN_PLACE_START; + if (placing->order_live) + trees_KeyOrder_drop(&placing->order); + doltgres_scan_option_drop(&placing->high); + doltgres_scan_option_drop(&placing->low); + doltgres_scan_option_drop(&placing->group); + placing->order_live = placing->moved = false; + if (frame->reading_live) + host_shared_Reader_drop(&frame->reading); + frame->reading_live = false; +} + +/* doltgres_scan_unit_placed_body finds a moved worker's place and stop in its unit in the newer base's tree. */ +static void +doltgres_scan_unit_placed_body(void *frame_) +{ + doltgres_scan_unit_placed_frame_t *frame = frame_; + DoltgresScanUnitPlacing *placing = &frame->placing; + const doltgres_OptionVecU8_t *from; + + placing->place.tag = DOLTGRES_SCAN_PLACE_START; + if (!host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_unit_placed_closure_0, placing, NULL, frame->error)) + return; + if (!placing->moved) + { + frame->ok = true; + return; + } + from = placing->group.has ? &placing->group : &placing->low; + if (!placing->has_tree) + placing->place = doltgres_scan_off(DOLTGRES_SCAN_DIRECTION_FORWARD); + else + { + if (!host_shared_read(frame->io, &frame->reading, frame->error)) + return; + frame->reading_live = true; + if (from->has) + { + doltgres_scan_place_read_t read = {{placing->tree, from->v.ptr, from->v.len, + DOLTGRES_SCAN_DIRECTION_FORWARD, &placing->order, placing->group.has}, .error = frame->error}; + + frame->stores->ops->with(frame->stores->self, frame->io, &frame->reading, doltgres_scan_place_read, + &read); + if (!read.ok) + return; + placing->place = read.place; + } + if (placing->high.has) + { + doltgres_scan_sought_read_t read = {{placing->tree, placing->high.v.ptr, placing->high.v.len, + &placing->order}, &placing->bound, frame->error}; + + frame->stores->ops->with(frame->stores->self, frame->io, &frame->reading, doltgres_scan_sought_read, + &read); + if (!read.ok) + return; + } + } + frame->ok = host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_unit_placed_closure_1, placing, NULL, + frame->error); +} + +/* + * doltgres_scan_unit_placed puts a parallel scan's worker that moved to a + * newer base than the shared one (rebased) at its place in its unit in the + * newer base's tree, outside the record, since a compared tree's order runs + * user code: past its group's key once it has one, as a serial read goes on, + * or else at the unit's low key, or the tree's start for unit 0; and stops it + * at the first entry at or past the unit's high key, which the newer tree + * need not hold, so each key is read by the unit whose edges hold it. A + * worker on the shared base, or with no unit, is left as it is. + */ +static bool +doltgres_scan_unit_placed(HostIoIo *io, const DoltgresScanStores *stores, HostOwnedHandle handle, HostError *error) +{ + doltgres_scan_unit_placed_frame_t frame = {.io = io, .stores = stores, .handle = handle, .error = error}; + + std_landing(&frame, doltgres_scan_unit_placed_body, doltgres_scan_unit_placed_pad); + doltgres_scan_unit_placed_pad(&frame); + return frame.ok; +} + +/* doltgres_scan_rebasing_frame_t is rebasing's state across its pad: its reader and the move, while live. */ +typedef struct doltgres_scan_rebasing_frame_t +{ + HostIoIo *io; + const DoltgresScanStores *stores; + const DoltgresScanUnder *under; + HostOwnedHandle handle; + HostSharedReader reading; + bool reading_live; + DoltgresScanRebased rebasing; + bool *moved; + HostError *error; + bool ok; +} doltgres_scan_rebasing_frame_t; + +/* doltgres_scan_rebasing_pad drops what rebasing holds. */ +static void +doltgres_scan_rebasing_pad(void *frame_) +{ + doltgres_scan_rebasing_frame_t *frame = frame_; + + doltgres_scan_Rebased_drop(&frame->rebasing); + if (frame->reading_live) + host_shared_Reader_drop(&frame->reading); + frame->reading_live = false; } -/* doltgres_scan_next_parallel_closure_4 is next_parallel's put of the delta's run in the record. */ -static bool -doltgres_scan_next_parallel_closure_4(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +/* doltgres_scan_rebasing_body moves the worker to the newest base under its reader. */ +static void +doltgres_scan_rebasing_body(void *frame_) { - doltgres_scan_run_put(scan, captures); - return true; + doltgres_scan_rebasing_frame_t *frame = frame_; + doltgres_scan_rebase_read_t rebase = {frame->under, frame->handle, &frame->rebasing, frame->error}; + + if (!host_shared_read(frame->io, &frame->reading, frame->error)) + return; + frame->reading_live = true; + frame->stores->ops->with(frame->stores->self, frame->io, &frame->reading, doltgres_scan_rebase_read, &rebase); + *frame->moved = rebase.ok && frame->rebasing.moved; + frame->ok = rebase.ok; } -/* doltgres_scan_next_parallel_closure_5 is next_parallel's making of the chosen group in the record, forward. */ +/* + * doltgres_scan_rebasing moves a parallel scan's worker to its storage's + * newest base under a reader of its own (rebased) and answers whether it + * moved. + */ static bool -doltgres_scan_next_parallel_closure_5(void *captures, HostIoIo *io, DoltgresScanScan *scan, HostError *error) +doltgres_scan_rebasing(HostIoIo *io, const DoltgresScanStores *stores, const DoltgresScanUnder *under, + HostOwnedHandle handle, bool *moved, HostError *error) { - doltgres_scan_chose_t *chose = captures; + doltgres_scan_rebasing_frame_t frame = {.io = io, .stores = stores, .under = under, .handle = handle, + .moved = moved, .error = error}; - return doltgres_scan_chosen(io, chose->under, scan, &chose->picked, DOLTGRES_SCAN_DIRECTION_FORWARD, error); + std_landing(&frame, doltgres_scan_rebasing_body, doltgres_scan_rebasing_pad); + doltgres_scan_rebasing_pad(&frame); + return frame.ok; } + /* * doltgres_scan_next_parallel_frame_t is next_parallel's state across its pad: * a unit's bounds, a turn's order, and a run's keys, while live. @@ -6125,6 +7190,7 @@ typedef struct doltgres_scan_next_parallel_frame_t TreesKeyOrder order; bool order_live; StdVec keys; + bool keys_live; DoltgresScanStep *result; HostError *error; bool ok; @@ -6136,6 +7202,9 @@ doltgres_scan_next_parallel_pad(void *frame_) { doltgres_scan_next_parallel_frame_t *frame = frame_; + if (frame->keys_live) + doltgres_scan_vecs_drop(&frame->keys); + frame->keys_live = false; if (frame->order_live) trees_KeyOrder_drop(&frame->order); frame->order_live = false; @@ -6149,20 +7218,43 @@ doltgres_scan_next_parallel_body(void *frame_) { doltgres_scan_next_parallel_frame_t *frame = frame_; doltgres_scan_turn_t turn = {frame->stores, frame->under, DOLTGRES_SCAN_DIRECTION_FORWARD}; + bool moved = false; for (;;) { DoltgresScanTurn step; + uint64 generation; if (!host_owned_TableDoltgresScanScan_withOptionBounds(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, doltgres_scan_next_parallel_closure_0, (void *) &frame->stores, NULL, &frame->begun, frame->error)) return; + if (moved || frame->begun.has) + { + if (!doltgres_scan_unit_placed(frame->io, frame->stores, frame->handle, frame->error)) + return; + moved = false; + } if (frame->begun.has) { - if (!doltgres_scan_unit_keys(frame->io, frame->under, &frame->begun.v, &frame->keys, frame->error)) + if (!doltgres_scan_unit_keys(frame->io, frame->under, frame->handle, &frame->begun.v, &frame->keys, + frame->error)) + return; + frame->keys_live = true; + if (!host_owned_TableDoltgresScanScan_withU64(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_generation_closure_0, NULL, NULL, &generation, + frame->error)) return; + if (txn_state_moved_past(frame->under->state, generation)) + { + doltgres_scan_next_parallel_pad(frame); + if (!doltgres_scan_rebasing(frame->io, frame->stores, frame->under, frame->handle, &moved, + frame->error)) + return; + continue; + } + frame->keys_live = false; if (!host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, doltgres_scan_next_parallel_closure_1, &frame->keys, doltgres_scan_keys_drop, frame->error)) @@ -6190,12 +7282,31 @@ doltgres_scan_next_parallel_body(void *frame_) frame->ok = true; return; } - if (step.tag == DOLTGRES_SCAN_TURN_RUN) + if (step.tag == DOLTGRES_SCAN_TURN_MOVED) + { + if (!doltgres_scan_rebasing(frame->io, frame->stores, frame->under, frame->handle, &moved, frame->error)) + return; + } + else if (step.tag == DOLTGRES_SCAN_TURN_RUN) { frame->order = step.u.run; frame->order_live = true; if (!doltgres_scan_run(frame->io, frame->under, &frame->order, &frame->keys, frame->error)) return; + frame->keys_live = true; + if (!host_owned_TableDoltgresScanScan_withU64(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, + doltgres_scan_generation_closure_0, NULL, NULL, &generation, + frame->error)) + return; + if (txn_state_moved_past(frame->under->state, generation)) + { + doltgres_scan_next_parallel_pad(frame); + if (!doltgres_scan_rebasing(frame->io, frame->stores, frame->under, frame->handle, &moved, + frame->error)) + return; + continue; + } + frame->keys_live = false; if (!host_owned_TableDoltgresScanScan_withUnit(&DOLTGRES_SCAN_SCANS, frame->io, frame->handle, doltgres_scan_next_parallel_closure_4, &frame->keys, doltgres_scan_keys_drop, frame->error)) @@ -6217,6 +7328,9 @@ doltgres_scan_next_parallel_body(void *frame_) doltgres_scan_chose_drop, frame->error)) return; doltgres_scan_next_parallel_pad(frame); + if (chose.moved && + !doltgres_scan_rebasing(frame->io, frame->stores, frame->under, frame->handle, &moved, frame->error)) + return; } } } @@ -6382,9 +7496,7 @@ doltgres_scan_tests_storage(void) static DoltgresScanTrees doltgres_scan_tests_trees(const TxnBaseBase *base) { - DoltgresScanTrees trees = {base->has_root, base->root, base->has_tuple_ids, base->tuple_ids, base->folded}; - - return trees; + return doltgres_scan_Trees_of(base, 0); } /* @@ -6514,18 +7626,20 @@ doltgres_scan_tests_le4(uint32 n) } /* - * doltgres_scan_tests_base builds a tree of rows rows in a store, each keyed - * by its number big-endian with its number's square as its value and - * stamped by its number plus one as its tuple id, and returns its root's name - * as a base's record would decode it. + * doltgres_scan_tests_built builds a tree of the rows numbered from one + * number up to another in a store, each keyed by its number big-endian with + * its number's square as its value and stamped by its number plus one as + * its tuple id, and returns its root's name as a base's record would decode + * it; its adds and its finish unwrap at a line and column each. */ static TreesStoreTree -doltgres_scan_tests_base(HostIoIo *io, TreesStoreMemoryStore *store, uint32 rows) +doltgres_scan_tests_built(HostIoIo *io, TreesStoreMemoryStore *store, uint32 from, uint32 to, uint32 add_line, + uint32 add_col, uint32 finished_line, uint32 finished_col) { TreesBuildBuilder builder; trees_build_Builder_new(TREES_STORE_MEMORY_STORE_NODE_STORE(store), 256, &builder); - for (uint32 n = 0; n < rows; n++) + for (uint32 n = from; n < to; n++) { StdVec key = doltgres_scan_tests_be4(n); StdVec value = doltgres_scan_tests_le4(n * n); @@ -6534,9 +7648,19 @@ doltgres_scan_tests_base(HostIoIo *io, TreesStoreMemoryStore *store, uint32 rows trees_build_Entry_new_VecU8VecU8(&key, &value, &entry); std_vec_push_Checked(&entry.stamps, &stamp, sizeof(ChunksTrailerStamp), _Alignof(ChunksTrailerStamp)); - doltgres_scan_tests_add(&builder, io, &entry, 1837, 105); + doltgres_scan_tests_add(&builder, io, &entry, add_line, add_col); } - return doltgres_scan_tests_finished(&builder, io, 1839, 53); + return doltgres_scan_tests_finished(&builder, io, finished_line, finished_col); +} + +/* + * doltgres_scan_tests_base builds a tree of rows rows in a store, as built + * builds the rows from 0, and returns its root's name. + */ +static TreesStoreTree +doltgres_scan_tests_base(HostIoIo *io, TreesStoreMemoryStore *store, uint32 rows) +{ + return doltgres_scan_tests_built(io, store, 0, rows, 1837, 105, 1839, 53); } /* doltgres_scan_tests_next reads a scan's next row, which unwraps at a line and column. */ @@ -6638,8 +7762,8 @@ doltgres_scan_tests_begin(HostIoIo *io, TreesStoreMemoryStore *store, uint64 blo HostError failed; std_slice_to_vec_Checked(&layout, keyed, keyed_len, DOLTGRES_SCAN_BYTES); - if (!doltgres_scan_scan_begin(io, TREES_STORE_MEMORY_STORE_NODE_STORE(store), blocks, base, &layout, table, &order, - &handle, &failed)) + if (!doltgres_scan_scan_begin(io, TREES_STORE_MEMORY_STORE_NODE_STORE(store), blocks, base, 0, &layout, table, + &order, &handle, &failed)) STD_RESULT_EXPECT_FAILED(line, col, DOLTGRES_SCAN_TESTS_UNWRAP, &failed, host_Error_Debug_fmt); return handle; } @@ -7084,11 +8208,13 @@ doltgres_scan_tests_fetch(HostIoIo *io, TreesStoreMemoryStore *store, const Dolt DoltgresScanRow *row, HostError *failed) { DoltgresScanEpochs epochs; + DoltgresScanResolved resolved = {0}; bool ok; doltgres_scan_Epochs_Default_default(&epochs); - ok = doltgres_scan_fetch(io, TREES_STORE_MEMORY_STORE_NODE_STORE(store), under, &epochs, + ok = doltgres_scan_fetch(io, TREES_STORE_MEMORY_STORE_NODE_STORE(store), under, &epochs, &resolved, doltgres_scan_tests_trees(base), &layout, tuple_id, has, row, failed); + doltgres_scan_Resolved_drop(&resolved); doltgres_scan_Epochs_drop(&epochs); return ok; } @@ -7652,7 +8778,7 @@ doltgres_scan_tests_begin_parallel(HostIoIo *io, DoltgresScanShared *shared, con HostError failed; std_slice_to_vec_Checked(&layout, keyed, keyed_len, DOLTGRES_SCAN_BYTES); - if (!doltgres_scan_begin_parallel(io, shared, &layout, 0, &order, &handle, &failed)) + if (!doltgres_scan_begin_parallel(io, shared, 0, &layout, 0, &order, &handle, &failed)) STD_RESULT_EXPECT_FAILED(line, col, DOLTGRES_SCAN_TESTS_UNWRAP, &failed, host_Error_Debug_fmt); return handle; } @@ -8808,7 +9934,7 @@ test_doltgres_scan_tests_a_compared_scan_merges_the_deltas_keys_in_the_tables_or under.has_serial = false; based = doltgres_scan_tests_based(tree, NULL, 1); std_slice_to_vec_Checked(&layout, keyed, 1, DOLTGRES_SCAN_BYTES); - if (!doltgres_scan_scan_begin(io, TREES_STORE_MEMORY_STORE_NODE_STORE(&store), 1, &based, &layout, 0, &order, + if (!doltgres_scan_scan_begin(io, TREES_STORE_MEMORY_STORE_NODE_STORE(&store), 1, &based, 0, &layout, 0, &order, &handle, &failed)) STD_RESULT_EXPECT_FAILED(2697, 95, DOLTGRES_SCAN_TESTS_UNWRAP, &failed, host_Error_Debug_fmt); w = writer.f0; @@ -9424,6 +10550,449 @@ test_doltgres_scan_tests_a_scan_with_no_sample_or_range_leaves_its_answers_as_th doltgres_scan_tests_run(test_doltgres_scan_tests_a_scan_with_no_sample_or_range_leaves_its_answers_as_they_were_closure_0); } +/* + * test_doltgres_scan_tests_a_scan_moves_to_a_newer_base_at_its_next_read_and_gives_every_row_once_closure_0 + * is the case's actor. + */ +static void +test_doltgres_scan_tests_a_scan_moves_to_a_newer_base_at_its_next_read_and_gives_every_row_once_closure_0(void *captures) +{ + HostIoIo *io = ((doltgres_scan_tests_case_t *) captures)->io; + static const uint8 keyed[] = {1}; + TreesStoreMemoryStore store = trees_store_MemoryStore_new(); + HostSharedHeld held = doltgres_scan_tests_made(io); + HostSharedRecord state = host_shared_Held_record(&held); + TreesStoreTree older = doltgres_scan_tests_base(io, &store, 4); + TreesStoreTree newer = doltgres_scan_tests_base(io, &store, 8); + HostXid writer = simc_begin(); + TxnStatusOutcome committed = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = 100}; + TxnStatusRoomed roomed; + bool took; + SimcTaken after_taken; + HostSnapshot after; + TxnBaseBase based; + TxnBaseBase installed; + uint64 generation; + HostOwnedHandle handle; + DoltgresScanUnder under; + StdVec rows; + StdVec rest; + HostError error; + + for (uint32 n = 4; n < 8; n++) + { + StdVec key = doltgres_scan_tests_be4(n); + StdVec tuple = doltgres_scan_tests_le4(n * n); + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_PUT}; + + op.u.put = doltgres_scan_tests_put(tuple.ptr, tuple.len, (uint64) n + 1); + doltgres_scan_tests_pushed(io, &held, key.ptr, key.len, writer, 0, &op); + std_vec_free_Checked(&tuple, DOLTGRES_SCAN_BYTES); + std_vec_free_Checked(&key, DOLTGRES_SCAN_BYTES); + } + if (!txn_status_take(io, writer, committed, &roomed, &took, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, DOLTGRES_SCAN_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + simc_commit(writer); + after_taken = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + after = simc_Taken_lent(&after_taken); + generation = txn_state_generation(state); + based = doltgres_scan_tests_based(older, NULL, 1); + handle = doltgres_scan_tests_begin(io, &store, 1, &based, keyed, 1, 0, __LINE__, 0); + under.state = state; + under.snapshot = &after; + under.has_serial = false; + std_vec_new(&rows, sizeof(DoltgresScanRow), _Alignof(DoltgresScanRow)); + for (int i = 0; i < 6; i++) + { + DoltgresScanRow row; + + STD_ASSERT(__LINE__, 0, + doltgres_scan_tests_next(io, &store, &under, handle, DOLTGRES_SCAN_DIRECTION_FORWARD, &row, + __LINE__, 0), + "next(io, &store, &under, handle, Direction::Forward).unwrap().is_some()"); + std_vec_push_Checked(&rows, &row, sizeof(DoltgresScanRow), _Alignof(DoltgresScanRow)); + } + installed = doltgres_scan_tests_based(newer, NULL, 1); + installed.folded = 200; + if (!txn_state_installed(HOST_IO_IO_TAKES_SHORT(io), &held, generation, &installed, NULL, NULL, &took, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, DOLTGRES_SCAN_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, took, "state::installed(io, &held, generation, &installed).unwrap()"); + for (uint32 n = 4; n < 8; n++) + { + StdVec key = doltgres_scan_tests_be4(n); + StdVec versions; + StdVec handles; + StdVec removed; + HostDeltaHold hold; + + if (!host_delta_hold(io, &hold, &error) || + !txn_memtable_table_versions(io, &hold, state, key.ptr, key.len, &versions, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, DOLTGRES_SCAN_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + std_vec_new(&handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + for (size_t v = 0; v < versions.len; v++) + std_vec_push_Checked(&handles, &STD_VEC_AT(&versions, TxnMemtableVersionVersion, v).handle, + sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + if (!txn_memtable_table_removed(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, state, key.ptr, key.len, handles.ptr, + handles.len, &removed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, DOLTGRES_SCAN_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + std_vec_free_Checked(&removed, sizeof(bool), _Alignof(bool)); + std_vec_free_Checked(&handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + std_vec_free_Checked(&key, DOLTGRES_SCAN_BYTES); + } + rest = doltgres_scan_tests_read(io, &store, &under, handle, DOLTGRES_SCAN_DIRECTION_FORWARD); + for (size_t i = 0; i < rest.len; i++) + std_vec_push_Checked(&rows, &STD_VEC_AT(&rest, DoltgresScanRow, i), sizeof(DoltgresScanRow), + _Alignof(DoltgresScanRow)); + std_vec_free_Checked(&rest, sizeof(DoltgresScanRow), _Alignof(DoltgresScanRow)); + STD_ASSERT(__LINE__, 0, doltgres_scan_tests_expected(&rows, 0, 8, false), "every row once across the install"); + doltgres_scan_tests_rows_drop(&rows); + rows = doltgres_scan_tests_read(io, &store, &under, handle, DOLTGRES_SCAN_DIRECTION_BACKWARD); + STD_ASSERT(__LINE__, 0, doltgres_scan_tests_expected(&rows, 0, 8, true), + "read(io, &store, &under, handle, Direction::Backward) == reversed(&expected(8))"); + doltgres_scan_tests_rows_drop(&rows); + doltgres_scan_tests_end(io, handle, __LINE__, 0); + simc_Taken_drop(&after_taken); + host_shared_Held_drop(&held); + trees_store_MemoryStore_drop(&store); +} + +/* + * Bugs rows 240 and 241: a scan that has read part of its base and of the + * delta when an install gives the storage a newer base holding the delta's + * committed rows, and a drop takes them out of the delta, moves to the newer + * base at its next read and goes on past its group's key, so it gives every + * row once, and its read back after gives them all once the other way. A + * C-only case. + */ +void +test_doltgres_scan_tests_a_scan_moves_to_a_newer_base_at_its_next_read_and_gives_every_row_once(void) +{ + doltgres_scan_tests_run(test_doltgres_scan_tests_a_scan_moves_to_a_newer_base_at_its_next_read_and_gives_every_row_once_closure_0); +} + +/* DOLTGRES_SCAN_TESTS_HELD_DATABASE is the held reader case's database. */ +#define DOLTGRES_SCAN_TESTS_HELD_DATABASE 9631 + +/* + * doltgres_scan_tests_held_t is the held reader case's actors' captures: the + * store, the older and newer bases' trees, the base generation the reader + * takes its base at, and the rows the reader read. + */ +typedef struct doltgres_scan_tests_held_t +{ + TreesStoreMemoryStore *store; + TreesStoreTree older; + TreesStoreTree newer; + uint64 generation; + StdVec *rows; +} doltgres_scan_tests_held_t; + +/* doltgres_scan_tests_held_storage returns the held reader case's storage. */ +static HostStorage +doltgres_scan_tests_held_storage(void) +{ + HostStorage storage = {HOST_PG_DEFAULT, DOLTGRES_SCAN_TESTS_HELD_DATABASE, 16384}; + + return storage; +} + +/* doltgres_scan_tests_held_state opens the held reader case's repository and its storage's state. */ +static bool +doltgres_scan_tests_held_state(HostIoIo *io, HostSharedHeld *result, HostError *error) +{ + HostStorage storage = doltgres_scan_tests_held_storage(); + RepoRepo repo; + HostSharedReader reading; + ChunksErrorError failed; + bool ok; + + if (!repo_Repo_open(io, DOLTGRES_SCAN_TESTS_HELD_DATABASE, HOST_PG_DEFAULT, &repo, &failed)) + { + *error = txn_base_storage(&failed); + return false; + } + ok = host_shared_read(io, &reading, error); + if (ok) + { + ok = txn_state_state(io, &repo, &reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, result, + error); + host_shared_Reader_drop(&reading); + } + repo_Repo_drop(&repo); + return ok; +} + +/* doltgres_scan_tests_held_reader is the held reader case's reader: it reads the scan of the older base to its end. */ +static bool +doltgres_scan_tests_held_reader(void *captures, HostIoIo *io, HostError *error) +{ + const doltgres_scan_tests_held_t *c = captures; + static const uint8 keyed[] = {1}; + TxnBaseBase based = doltgres_scan_tests_based(c->older, NULL, 1); + DoltgresScanStores stores = doltgres_scan_tests_stores(c->store); + DoltgresScanLayout layout = doltgres_scan_tests_layout(keyed, 1, 0); + HostSharedHeld held; + SimcTaken taken; + HostSnapshot snapshot; + DoltgresScanUnder under = {.has_serial = false}; + HostOwnedHandle handle; + bool ok; + + if (!doltgres_scan_tests_held_state(io, &held, error)) + { + doltgres_scan_Layout_drop(&layout); + return false; + } + taken = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + snapshot = simc_Taken_lent(&taken); + under.state = host_shared_Held_record(&held); + under.snapshot = &snapshot; + ok = doltgres_scan_scan_begin(io, TREES_STORE_MEMORY_STORE_NODE_STORE(c->store), 1, &based, c->generation, + &layout.keyed, 0, &layout.order, &handle, error); + for (;;) + { + DoltgresScanRow row; + bool has; + + if (!ok) + break; + ok = doltgres_scan_next(io, &stores, &under, handle, DOLTGRES_SCAN_DIRECTION_FORWARD, &has, &row, error); + if (!ok || !has) + break; + std_vec_push_Checked(c->rows, &row, sizeof(DoltgresScanRow), _Alignof(DoltgresScanRow)); + } + ok = ok && doltgres_scan_scan_end(io, handle, error); + simc_Taken_drop(&taken); + host_shared_Held_drop(&held); + return ok; +} + +/* doltgres_scan_tests_held_folder is the held reader case's folder: it installs the newer base and drops the spills. */ +static bool +doltgres_scan_tests_held_folder(void *captures, HostIoIo *io, HostError *error) +{ + const doltgres_scan_tests_held_t *c = captures; + TxnBaseBase installed = doltgres_scan_tests_based(c->newer, NULL, 1); + HostSharedHeld held; + bool took = false; + bool ok; + + installed.folded = 200; + if (!doltgres_scan_tests_held_state(io, &held, error)) + return false; + ok = txn_state_installed(HOST_IO_IO_TAKES_SHORT(io), &held, c->generation, &installed, NULL, NULL, &took, error); + STD_ASSERT(__LINE__, 0, !ok || took, "state::installed(io, &held, generation, &installed)?"); + ok = ok && txn_spill_folded(io, host_shared_Held_record(&held), 200, error); + host_shared_Held_drop(&held); + return ok; +} + +/* + * Bugs rows 240 and 241's check after the delta read: a scan's first read, + * held inside its read of the delta before it reads the spill files that + * hold the delta's committed rows, all below its base's first entry, while + * an install gives the storage a newer base holding them and a drop takes + * their files out, finds once it read the delta that the base moved, moves + * to the newer base, and gives every row once; with the check before the + * delta read it pairs its base with the dropped delta and misses them. A + * C-only case. + */ +void +test_doltgres_scan_tests_a_scan_held_inside_its_delta_read_across_an_install_and_drop_gives_every_row_once(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostStorage storage = doltgres_scan_tests_held_storage(); + TreesStoreMemoryStore store = trees_store_MemoryStore_new(); + StdVec rows; + doltgres_scan_tests_held_t captures = {.store = &store, .rows = &rows}; + SimcActor reader = {doltgres_scan_tests_held_reader, &captures}; + SimcActor folder = {doltgres_scan_tests_held_folder, &captures}; + SimcStep steps[2] = {{"folder", 6, PG_UINT64_MAX}, {"reader", 6, PG_UINT64_MAX}}; + SimcEnding returned = {.tag = SIMC_ENDING_RETURNED}; + TxnStatusOutcome committed = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = 100}; + TxnStatusRoomed roomed; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostXid writer; + ChunksErrorError created; + HostError error; + SimcRun run; + SimcReport report; + bool spilled; + bool took; + bool has_count; + uint64 count; + + captures.older = doltgres_scan_tests_built(io, &store, 8, 12, __LINE__, 0, __LINE__, 0); + captures.newer = doltgres_scan_tests_built(io, &store, 0, 12, __LINE__, 0, __LINE__, 0); + if (!repo_Repo_create(io, DOLTGRES_SCAN_TESTS_HELD_DATABASE, HOST_PG_DEFAULT, &repo, &created)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, DOLTGRES_SCAN_TESTS_UNWRAP, &created, chunks_error_Error_Debug_fmt); + if (!host_shared_read(io, &reading, &error) || + !txn_state_state(io, &repo, &reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &held, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, DOLTGRES_SCAN_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + writer = simc_begin(); + for (uint32 n = 0; n < 8; n++) + { + StdVec key = doltgres_scan_tests_be4(n); + StdVec tuple = doltgres_scan_tests_le4(n * n); + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_PUT}; + + op.u.put = doltgres_scan_tests_put(tuple.ptr, tuple.len, (uint64) n + 1); + doltgres_scan_tests_pushed(io, &held, key.ptr, key.len, writer, 0, &op); + std_vec_free_Checked(&tuple, DOLTGRES_SCAN_BYTES); + std_vec_free_Checked(&key, DOLTGRES_SCAN_BYTES); + } + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, writer.f0, &spilled, &error) || + !txn_status_take(io, writer, committed, &roomed, &took, &error) || + !txn_status_add(io, writer, 8, &has_count, &count, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, DOLTGRES_SCAN_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, spilled, "spill::spilled(io, &storage, Of::Rows, writer.0).unwrap()"); + simc_commit(writer); + captures.generation = txn_state_generation(host_shared_Held_record(&held)); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + std_vec_new(&rows, sizeof(DoltgresScanRow), _Alignof(DoltgresScanRow)); + run = simc_Run_new(5); + simc_Run_actor(&run, "reader", 6, reader); + simc_Run_actor(&run, "folder", 6, folder); + simc_Run_until(&run, "reader", 6, "doltgres-spill-slots-read"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + STD_ASSERT(__LINE__, 0, report.outcome.tag == SIMC_OUTCOME_ENDED, "the run ended"); + for (size_t k = 0; k < report.outcome.u.ended.len; k++) + STD_ASSERT(__LINE__, 0, + simc_Ending_PartialEq_eq(&STD_VEC_AT(&report.outcome.u.ended, simc_TupleStringEnding_t, k).f1, + &returned), + "every actor returned Ok(())"); + STD_ASSERT(__LINE__, 0, doltgres_scan_tests_expected(&rows, 0, 12, false), + "every row once across the install and drop"); + doltgres_scan_tests_rows_drop(&rows); + simc_Report_drop(&report); + trees_store_MemoryStore_drop(&store); + simc_TestIo_drop(&test_io); +} + +/* DOLTGRES_SCAN_TESTS_SPREAD is how many rows the memo case spreads over its four keyless entries. */ +#define DOLTGRES_SCAN_TESTS_SPREAD 4000 + +/* + * doltgres_scan_tests_spread builds a keyless table's base of four entries + * whose copies take tuple ids in turn, as a statement inserting four values + * in turn writes them, so that each copy is a run of its own, the copy of + * tuple id t of transaction txn plus t / 1000, each entry's row the key's + * byte plus a shift, with its tuple id tree. + */ +static TxnBaseBase +doltgres_scan_tests_spread(HostIoIo *io, TreesStoreMemoryStore *store, uint64 txn, uint8 shift) +{ + static uint64 tuple_ids[DOLTGRES_SCAN_TESTS_SPREAD]; + static StdVec keys[DOLTGRES_SCAN_TESTS_SPREAD]; + TreesBuildBuilder builder; + TreesStoreTree tree; + TreesStoreTree ids; + + trees_build_Builder_new(TREES_STORE_MEMORY_STORE_NODE_STORE(store), 256, &builder); + for (uint8 byte = 1; byte <= 4; byte++) + { + StdVec key; + StdVec value; + TreesBuildEntry entry; + uint8 row = byte + shift; + + std_vec_u8_from_elem(&key, byte, 16); + txn_counted_keyless_value(DOLTGRES_SCAN_TESTS_SPREAD / 4, &row, 1, &value); + trees_build_Entry_new_VecU8VecU8(&key, &value, &entry); + for (uint64 tuple = byte; tuple <= DOLTGRES_SCAN_TESTS_SPREAD; tuple += 4) + { + ChunksTrailerStamp stamp = {txn + tuple / 1000, tuple}; + + std_vec_push_Checked(&entry.stamps, &stamp, sizeof(ChunksTrailerStamp), _Alignof(ChunksTrailerStamp)); + } + doltgres_scan_tests_add(&builder, io, &entry, __LINE__, 9); + } + tree = doltgres_scan_tests_finished(&builder, io, __LINE__, 9); + for (size_t i = 0; i < DOLTGRES_SCAN_TESTS_SPREAD; i++) + { + tuple_ids[i] = i + 1; + std_vec_u8_from_elem(&keys[i], (uint8) (i % 4 + 1), 16); + } + ids = doltgres_scan_tests_pairs(io, store, tuple_ids, keys, DOLTGRES_SCAN_TESTS_SPREAD); + return doltgres_scan_tests_based(tree, &ids, 1); +} + +/* doltgres_scan_tests_memo_fetch fetches a tuple id of a spread base through a memo, asserting its row's key, row, and stamp at a line. */ +static void +doltgres_scan_tests_memo_fetch(HostIoIo *io, doltgres_scan_tests_under_t *u, const TxnBaseBase *base, + DoltgresScanEpochs *epochs, DoltgresScanResolved *resolved, uint64 tuple, uint64 txn, + uint8 shift, uint32 line) +{ + DoltgresScanLayout layout = doltgres_scan_tests_layout(NULL, 0, 0); + uint8 key[16]; + uint8 byte = (uint8) ((tuple - 1) % 4 + 1); + uint8 expected = byte + shift; + DoltgresScanRow row; + HostError failed; + bool has; + + memset(key, byte, sizeof(key)); + if (!doltgres_scan_fetch(io, TREES_STORE_MEMORY_STORE_NODE_STORE(&u->store), &u->under, epochs, resolved, + doltgres_scan_tests_trees(base), &layout, tuple, &has, &row, &failed)) + STD_RESULT_EXPECT_FAILED(line, 13, DOLTGRES_SCAN_TESTS_UNWRAP, &failed, host_Error_Debug_fmt); + if (!has) + STD_OPTION_UNWRAP_FAILED(line, 13); + STD_ASSERT(line, 9, + doltgres_scan_tests_bytes_is(&row.key, key, sizeof(key)) && + doltgres_scan_tests_bytes_is(&row.tuple, &expected, 1) && row.stamp.tuple == tuple && + row.stamp.txn == txn + tuple / 1000, + "(row.key, row.tuple, row.stamp) == (vec![byte; 16], vec![byte + shift], (txn + tuple / 1000, tuple))"); + doltgres_scan_Row_drop(&row); +} + +/* test_doltgres_scan_tests_fetches_through_one_memo_give_each_keyless_copys_own_stamp_closure_0 is the case's actor. */ +static void +test_doltgres_scan_tests_fetches_through_one_memo_give_each_keyless_copys_own_stamp_closure_0(void *captures) +{ + HostIoIo *io = ((doltgres_scan_tests_case_t *) captures)->io; + doltgres_scan_tests_under_t u; + DoltgresScanEpochs epochs; + DoltgresScanResolved resolved = {0}; + TxnBaseBase first; + TxnBaseBase second; + + doltgres_scan_tests_under(io, &u); + doltgres_scan_Epochs_Default_default(&epochs); + first = doltgres_scan_tests_spread(io, &u.store, 100, 0); + for (uint64 tuple = 1; tuple <= DOLTGRES_SCAN_TESTS_SPREAD; tuple++) + doltgres_scan_tests_memo_fetch(io, &u, &first, &epochs, &resolved, tuple, 100, 0, __LINE__); + second = doltgres_scan_tests_spread(io, &u.store, 900, 10); + for (uint64 tuple = 1; tuple <= DOLTGRES_SCAN_TESTS_SPREAD; tuple += 333) + doltgres_scan_tests_memo_fetch(io, &u, &second, &epochs, &resolved, tuple, 900, 10, __LINE__); + doltgres_scan_tests_memo_fetch(io, &u, &first, &epochs, &resolved, 7, 100, 0, __LINE__); + doltgres_scan_Resolved_drop(&resolved); + doltgres_scan_Epochs_drop(&epochs); + doltgres_scan_tests_under_drop(&u); +} + +/* + * test_doltgres_scan_tests_fetches_through_one_memo_give_each_keyless_copys_own_stamp: + * bugs row 346: fetches of every tuple id of a keyless table whose copies + * take tuple ids in turn, through one memo of the leaf they last resolved a + * row in, each give the copy's own key and stamp; a second base of the same + * keys, other rows and other transactions' copies, fetched through the same + * memo, gives its own rows and stamps, never the first base's kept leaf's, + * and the first again its own. + */ +void +test_doltgres_scan_tests_fetches_through_one_memo_give_each_keyless_copys_own_stamp(void) +{ + doltgres_scan_tests_run(test_doltgres_scan_tests_fetches_through_one_memo_give_each_keyless_copys_own_stamp_closure_0); +} + #endif /* DOLTGRES_TEST_DOLTGRES */ #endif /* FRONTEND */ diff --git a/src/doltgres/doltgres/in_progress.c b/src/doltgres/doltgres/in_progress.c index 3052110682115..60493fcb9b73c 100644 --- a/src/doltgres/doltgres/in_progress.c +++ b/src/doltgres/doltgres/in_progress.c @@ -25,6 +25,9 @@ #include "postgres.h" #include "access/subtrans.h" +#include "access/transam.h" +#include "access/xlog.h" +#include "access/xlogutils.h" #include "doltgres/in_progress.h" #include "storage/proc.h" #include "storage/procarray.h" @@ -33,7 +36,10 @@ * doltgres_xid_in_progress says whether a transaction or a subtransaction is * in progress, as TransactionIdIsInProgress does, also for a caller outside the * proc array: the transaction in the slot at the caller's offset, its cached - * subtransactions, and, past its cache, the ones pg_subtrans records. + * subtransactions, and, past its cache, the ones pg_subtrans records. In + * recovery with no standby's bookkeeping, where redo marks pg_xact but never + * advances latestCompletedXid, it answers from pg_xact alone, an xid below + * the oldest one pg_xact keeps being one that ended (bugs row 340). */ bool doltgres_xid_in_progress(TransactionId xid) @@ -43,6 +49,16 @@ doltgres_xid_in_progress(TransactionId xid) bool found = false; bool overflowed = false; + if (standbyState == STANDBY_DISABLED && RecoveryInProgress()) + { + if (!TransactionIdIsNormal(xid)) + return false; + LWLockAcquire(XactTruncationLock, LW_SHARED); + found = !TransactionIdPrecedes(xid, TransamVariables->oldestClogXid) && !TransactionIdDidCommit(xid) && + !TransactionIdDidAbort(xid); + LWLockRelease(XactTruncationLock); + return found; + } if (TransactionIdIsInProgress(xid)) return true; if (MyProc == NULL || !TransactionIdIsNormal(xid)) diff --git a/src/doltgres/host/c/delta/directory.c b/src/doltgres/host/c/delta/directory.c index 29cfa5260f3dc..3108772fc5336 100644 --- a/src/doltgres/host/c/delta/directory.c +++ b/src/doltgres/host/c/delta/directory.c @@ -56,6 +56,8 @@ static void host_delta_directory_held(HostDeltaDirectoryDirectory directory, uin static bool host_delta_directory_raise(HostDeltaDirectoryDirectory directory, uint64 stamp); static bool host_delta_directory_owns(HostDeltaDirectoryDirectory directory, uint32 slot, HostDeltaDirectoryOwner owner, HostError *error); +static void host_delta_directory_freed_slot(HostDeltaDirectoryDirectory directory, uint32 slot, uint64 loaded, + uint64 oldest, bool *has_kept, uint64 *kept, bool *keep); /* host_delta_directory_internal returns the fork's internal error with a message. */ static HostError @@ -994,11 +996,47 @@ host_delta_directory_seal(HostDeltaDirectoryDirectory directory, return true; } +/* + * host_delta_directory_listed_record pushes an aged record its retirement + * just retired onto its kind's retired list, by a compare-and-swap of the + * list's head (bugs row 309). + */ +static void +host_delta_directory_listed_record(HostDeltaDirectoryDirectory directory, uint32 kind, uint32 index) +{ + for (;;) + { + uint64 head = directory.ops->record_retired_head(directory.self, kind); + + directory.ops->set_record_link(directory.self, kind, index, head); + if (directory.ops->record_retired_swap(directory.self, kind, head, (uint64) index + 1)) + return; + } +} + +/* + * host_delta_directory_listed pushes a slot its retirement just retired onto + * the retired list, by a compare-and-swap of the list's head (bugs row 309). + */ +static void +host_delta_directory_listed(HostDeltaDirectoryDirectory directory, uint32 slot) +{ + for (;;) + { + uint64 head = directory.ops->retired_head(directory.self); + + directory.ops->set_link(directory.self, slot, (uint32) head); + if (directory.ops->retired_swap(directory.self, head, (uint64) slot + 1)) + return; + } +} + /* * host_delta_directory_retire retires one batch of records and names under * one take of the epoch lock: it stores the batch's one stamp and then the * retired state in each record that is still published at its location and - * each page that is still live at its name, and then advances the epoch. + * each page that is still live at its name, pushing each onto its retired + * list, and then advances the epoch. */ void host_delta_directory_retire(HostDeltaDirectoryDirectory directory, @@ -1032,7 +1070,8 @@ host_delta_directory_retire(HostDeltaDirectoryDirectory directory, { directory.ops->set_stamp(directory.self, kind, index, stamp); directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_STAMPING); - directory.ops->record_swap(directory.self, kind, index, loaded, retired); + if (directory.ops->record_swap(directory.self, kind, index, loaded, retired)) + host_delta_directory_listed_record(directory, kind, index); } } for (size_t i = 0; i < names_len; i++) @@ -1055,7 +1094,8 @@ host_delta_directory_retire(HostDeltaDirectoryDirectory directory, { directory.ops->store(directory.self, slot, HOST_DELTA_DIRECTORY_FIELD_STAMP, stamp); directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_STAMPING); - directory.ops->swap(directory.self, slot, loaded, retired); + if (directory.ops->swap(directory.self, slot, loaded, retired)) + host_delta_directory_listed(directory, slot); } } directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_STAMPED); @@ -1065,12 +1105,55 @@ host_delta_directory_retire(HostDeltaDirectoryDirectory directory, } /* - * host_delta_directory_pass is the free pass: with the oldest published epoch - * loaded once, it walks every aged kind's records in the table's order and - * then the page directory, and claims each retired record or slot whose stamp - * that epoch has passed and frees it at its next generation, a page's body - * given back to its block first, each of which counts as a free. A pass that - * loses a claim does nothing with that record or slot. A pass that left no + * host_delta_directory_freed_slot frees a retired slot whose stamp the oldest + * epoch has passed, a page's body given back to its block first, counting the + * free, or else sets keep and lowers the lowest stamp kept; a slot not + * retired is neither. Its claim can't lose, since only a pass's claim takes a + * word out of retired and passes run one at a time. + */ +static void +host_delta_directory_freed_slot(HostDeltaDirectoryDirectory directory, uint32 slot, uint64 loaded, uint64 oldest, + bool *has_kept, uint64 *kept, bool *keep) +{ + uint64 freeing; + uint64 stamp; + uint64 released; + bool claimed PG_USED_FOR_ASSERTS_ONLY; + + *keep = false; + if (!host_delta_page_reap(loaded, &freeing)) + return; + stamp = directory.ops->load(directory.self, slot, HOST_DELTA_DIRECTORY_FIELD_STAMP); + if (!host_delta_page_passed(stamp, oldest)) + { + *kept = *has_kept ? std_min_u64(*kept, stamp) : stamp; + *has_kept = true; + *keep = true; + return; + } + claimed = directory.ops->swap(directory.self, slot, loaded, freeing); + Assert(claimed); + directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_CLAIMED); + host_delta_blocks_give(directory, directory.ops->load(directory.self, slot, HOST_DELTA_DIRECTORY_FIELD_BODY)); + directory.ops->store(directory.self, slot, HOST_DELTA_DIRECTORY_FIELD_BODY, 0); + host_delta_directory_freed(directory, slot, false); + released = host_delta_directory_emptied(directory, slot, freeing); + directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_HINTING); + host_delta_directory_hinted(directory, slot, released); + directory.ops->freed(directory.self); +} + +/* + * host_delta_directory_pass is the free pass, one at a time under the pass's + * lock: with the oldest published epoch loaded once, it takes each aged + * kind's retired list whole, in the table's order, and then the slots', and + * claims each retired record or slot whose stamp that epoch has passed and + * frees it at its next generation, a page's body given back to its block + * first, each of which counts as a free, pushing the ones it keeps back onto + * their list as one chain, so its work is the retirements it finds and never + * the directory's or a pool's size (bugs row 309). Its claims can't lose, + * since only a pass's claim takes a word out of retired, the pool's doors + * refusing an aged kind, and passes run one at a time. A pass that left no * retired record or slot unfreed clears the count of retirements it loaded, * unless a retirement since changed it. It loads the current epoch before the * oldest, and records the lowest stamp it kept (held). @@ -1084,6 +1167,7 @@ host_delta_directory_pass(HostDeltaDirectoryDirectory directory) uint64 current; uint64 oldest; + directory.ops->lock_pass(directory.self); directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_PASSING); left = directory.ops->left(directory.self); current = directory.ops->epoch(directory.self); @@ -1091,12 +1175,21 @@ host_delta_directory_pass(HostDeltaDirectoryDirectory directory) directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_OLDEST); for (uint32 kind = 0, kind_len = directory.ops->aged(directory.self); kind < kind_len; kind++) { - for (uint32 index = 0, index_len = directory.ops->records(directory.self, kind); index < index_len; index++) + uint64 at = directory.ops->record_retired_head(directory.self, kind); + uint64 first = 0; + uint32 last = 0; + + while (!directory.ops->record_retired_swap(directory.self, kind, at, 0)) + at = directory.ops->record_retired_head(directory.self, kind); + while (at != 0) { + uint32 index = (uint32) (at - 1); uint64 loaded = directory.ops->record_word(directory.self, kind, index); uint64 freeing; uint64 stamp; + bool claimed PG_USED_FOR_ASSERTS_ONLY; + at = directory.ops->record_link(directory.self, kind, index); if (!host_delta_page_reap(loaded, &freeing)) continue; stamp = directory.ops->stamp(directory.self, kind, index); @@ -1104,45 +1197,62 @@ host_delta_directory_pass(HostDeltaDirectoryDirectory directory) { kept = has_kept ? std_min_u64(kept, stamp) : stamp; has_kept = true; + if (first == 0) + last = index; + directory.ops->set_record_link(directory.self, kind, index, first); + first = (uint64) index + 1; continue; } - if (!directory.ops->record_swap(directory.self, kind, index, loaded, freeing)) - continue; + claimed = directory.ops->record_swap(directory.self, kind, index, loaded, freeing); + Assert(claimed); directory.ops->record_swap(directory.self, kind, index, freeing, host_delta_page_released(freeing)); directory.ops->freed(directory.self); } + while (first != 0) + { + uint64 head = directory.ops->record_retired_head(directory.self, kind); + + directory.ops->set_record_link(directory.self, kind, last, head); + if (directory.ops->record_retired_swap(directory.self, kind, head, first)) + break; + } } directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_WALKED); - for (uint32 slot = 0, slot_len = directory.ops->slots(directory.self); slot < slot_len; slot++) { - uint64 loaded = directory.ops->word(directory.self, slot); - uint64 freeing; - uint64 stamp; - uint64 released; + uint64 at = directory.ops->retired_head(directory.self); + uint32 first = 0; + uint32 last = 0; - if (!host_delta_page_reap(loaded, &freeing)) - continue; - stamp = directory.ops->load(directory.self, slot, HOST_DELTA_DIRECTORY_FIELD_STAMP); - if (!host_delta_page_passed(stamp, oldest)) + while (!directory.ops->retired_swap(directory.self, at, 0)) + at = directory.ops->retired_head(directory.self); + while (at != 0) { - kept = has_kept ? std_min_u64(kept, stamp) : stamp; - has_kept = true; - continue; + uint32 slot = (uint32) (at - 1); + bool keep; + + at = directory.ops->link(directory.self, slot); + host_delta_directory_freed_slot(directory, slot, directory.ops->word(directory.self, slot), oldest, + &has_kept, &kept, &keep); + if (!keep) + continue; + if (first == 0) + last = slot; + directory.ops->set_link(directory.self, slot, first); + first = slot + 1; + } + while (first != 0) + { + uint64 head = directory.ops->retired_head(directory.self); + + directory.ops->set_link(directory.self, last, (uint32) head); + if (directory.ops->retired_swap(directory.self, head, (uint64) first)) + break; } - if (!directory.ops->swap(directory.self, slot, loaded, freeing)) - continue; - directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_CLAIMED); - host_delta_blocks_give(directory, directory.ops->load(directory.self, slot, HOST_DELTA_DIRECTORY_FIELD_BODY)); - directory.ops->store(directory.self, slot, HOST_DELTA_DIRECTORY_FIELD_BODY, 0); - host_delta_directory_freed(directory, slot, false); - released = host_delta_directory_emptied(directory, slot, freeing); - directory.ops->stop(directory.self, HOST_DELTA_DIRECTORY_POINT_HINTING); - host_delta_directory_hinted(directory, slot, released); - directory.ops->freed(directory.self); } if (!has_kept) directory.ops->settle(directory.self, left); host_delta_directory_held(directory, current, oldest, has_kept ? kept : 0); + directory.ops->unlock_pass(directory.self); } /* host_delta_directory_rank orders kept stamps, zero, for none, after every stamp. */ @@ -1377,4 +1487,643 @@ test_host_delta_directory_tests_a_words_index_is_below_its_pages_count(void) host_delta_directory_tests_ResultTupleArray5U8_Debug_fmt); } +/* HOST_DELTA_DIRECTORY_TESTS_MOST is the most slots, and aged records, a test's directory holds. */ +#define HOST_DELTA_DIRECTORY_TESTS_MOST 4096 + +/* + * host_delta_directory_tests_Directory_t is a test's directory in process + * memory: its slots and their words and links, one aged kind's records, + * stamps, and links, the lists' heads, the header's counts, the epochs, and + * the count of the slot and record words loaded while a case counts them. + */ +typedef struct host_delta_directory_tests_Directory_t +{ + uint32 slots; + uint64 words[HOST_DELTA_DIRECTORY_TESTS_MOST]; + uint64 fields[HOST_DELTA_DIRECTORY_TESTS_MOST][7]; + uint32 links[HOST_DELTA_DIRECTORY_TESTS_MOST]; + uint64 records[HOST_DELTA_DIRECTORY_TESTS_MOST]; + uint64 stamps[HOST_DELTA_DIRECTORY_TESTS_MOST]; + uint64 record_links[HOST_DELTA_DIRECTORY_TESTS_MOST]; + uint64 head; + uint64 record_head; + uint64 frees; + uint64 left; + uint64 kept; + uint64 overshoot; + uint64 epoch; + uint64 oldest; + /* passing is whether visits counts the slot and record words loaded. */ + bool passing; + uint64 visits; +} host_delta_directory_tests_Directory_t; + +/* HOST_DELTA_DIRECTORY_TESTS_AGED is the id of a test directory's one aged kind. */ +#define HOST_DELTA_DIRECTORY_TESTS_AGED UINT64CONST(0x74657374) + +/* host_delta_directory_tests_directory is the directory a case builds, too large for a stack. */ +static host_delta_directory_tests_Directory_t host_delta_directory_tests_directory; + +/* HOST_DELTA_DIRECTORY_TESTS_SELF is a test directory's ops' self as the directory it is. */ +#define HOST_DELTA_DIRECTORY_TESTS_SELF(self) ((host_delta_directory_tests_Directory_t *) (self)) + +/* host_delta_directory_tests_slots is a test directory's count of slots. */ +static uint32 +host_delta_directory_tests_slots(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->slots; +} + +/* host_delta_directory_tests_budget is a test directory's budget, one 64 KiB block for each slot. */ +static uint64 +host_delta_directory_tests_budget(void *self) +{ + return (uint64) HOST_DELTA_DIRECTORY_TESTS_SELF(self)->slots * HOST_DELTA_PAGE_BLOCK; +} + +/* host_delta_directory_tests_word loads a slot's word, counting it while a case counts. */ +static uint64 +host_delta_directory_tests_word(void *self, uint32 slot) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->passing) + d->visits++; + return d->words[slot]; +} + +/* host_delta_directory_tests_swap stores a slot's word over the one loaded. */ +static bool +host_delta_directory_tests_swap(void *self, uint32 slot, uint64 loaded, uint64 word) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->words[slot] != loaded) + return false; + d->words[slot] = word; + return true; +} + +/* host_delta_directory_tests_load loads one of a slot's other words. */ +static uint64 +host_delta_directory_tests_load(void *self, uint32 slot, HostDeltaDirectoryField field) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->fields[slot][field]; +} + +/* host_delta_directory_tests_store stores one of a slot's other words. */ +static void +host_delta_directory_tests_store(void *self, uint32 slot, HostDeltaDirectoryField field, uint64 value) +{ + HOST_DELTA_DIRECTORY_TESTS_SELF(self)->fields[slot][field] = value; +} + +/* host_delta_directory_tests_blocks is a test directory's count of blocks, which no case takes. */ +static uint32 +host_delta_directory_tests_blocks(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->slots; +} + +/* host_delta_directory_tests_region loads a region word, none of which a case sets. */ +static uint64 +host_delta_directory_tests_region(void *self, HostDeltaBlocksRegion word) +{ + return 0; +} + +/* host_delta_directory_tests_set_region stores a region word, which no case reads. */ +static void +host_delta_directory_tests_set_region(void *self, HostDeltaBlocksRegion word, uint64 value) +{ +} + +/* host_delta_directory_tests_block loads a block's word, none of which a case sets. */ +static uint64 +host_delta_directory_tests_block(void *self, uint32 block, HostDeltaBlocksBlock word) +{ + return 0; +} + +/* host_delta_directory_tests_set_block stores a block's word, which no case reads. */ +static void +host_delta_directory_tests_set_block(void *self, uint32 block, HostDeltaBlocksBlock word, uint64 value) +{ +} + +/* host_delta_directory_tests_nothing does nothing, as a test directory's locks and listings do. */ +static void +host_delta_directory_tests_nothing(void *self) +{ +} + +/* host_delta_directory_tests_frees is a test directory's count of frees. */ +static uint64 +host_delta_directory_tests_frees(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->frees; +} + +/* host_delta_directory_tests_freed counts a free. */ +static void +host_delta_directory_tests_freed(void *self) +{ + HOST_DELTA_DIRECTORY_TESTS_SELF(self)->frees++; +} + +/* host_delta_directory_tests_left is a test directory's count of retirements. */ +static uint64 +host_delta_directory_tests_left(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->left; +} + +/* host_delta_directory_tests_leave counts a retirement. */ +static void +host_delta_directory_tests_leave(void *self) +{ + HOST_DELTA_DIRECTORY_TESTS_SELF(self)->left++; +} + +/* host_delta_directory_tests_settle clears the count of retirements while it holds the one loaded. */ +static void +host_delta_directory_tests_settle(void *self, uint64 loaded) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->left == loaded) + d->left = 0; +} + +/* host_delta_directory_tests_kept is a test directory's kept stamp. */ +static uint64 +host_delta_directory_tests_kept(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->kept; +} + +/* host_delta_directory_tests_kept_swap stores the kept stamp over the one loaded. */ +static bool +host_delta_directory_tests_kept_swap(void *self, uint64 loaded, uint64 word) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->kept != loaded) + return false; + d->kept = word; + return true; +} + +/* host_delta_directory_tests_zero answers 0 for a count no case moves. */ +static uint64 +host_delta_directory_tests_zero_count(void *self) +{ + return 0; +} + +/* host_delta_directory_tests_count_waiter counts nothing, since no case waits. */ +static void +host_delta_directory_tests_count_waiter(void *self, bool add) +{ +} + +/* host_delta_directory_tests_overshoot is a test directory's overshoot word. */ +static uint64 +host_delta_directory_tests_overshoot(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->overshoot; +} + +/* host_delta_directory_tests_overshoot_swap stores the overshoot word over the one loaded. */ +static bool +host_delta_directory_tests_overshoot_swap(void *self, uint64 loaded, uint64 word) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->overshoot != loaded) + return false; + d->overshoot = word; + return true; +} + +/* host_delta_directory_tests_hand is a test directory's hand, its first slot. */ +static uint32 +host_delta_directory_tests_hand(void *self) +{ + return 0; +} + +/* host_delta_directory_tests_hint is a word of a test directory's free hint, which holds no free slot. */ +static uint64 +host_delta_directory_tests_hint(void *self, uint32 word) +{ + return 0; +} + +/* host_delta_directory_tests_hint_slot sets or clears no bit, since no case frees a slot. */ +static void +host_delta_directory_tests_hint_slot(void *self, uint32 slot) +{ +} + +/* host_delta_directory_tests_spent is a test directory's count of spent slots, none. */ +static uint64 +host_delta_directory_tests_spent(void *self, bool over) +{ + return 0; +} + +/* host_delta_directory_tests_spend counts no spent slot, since no case spends one. */ +static void +host_delta_directory_tests_spend(void *self, bool over) +{ +} + +/* host_delta_directory_tests_lock takes a test directory's allocation lock, which no other process takes. */ +static bool +host_delta_directory_tests_lock(void *self, const void *raise, HostError *error) +{ + return true; +} + +/* host_delta_directory_tests_zero_body zeroes no body, since no case takes one. */ +static void +host_delta_directory_tests_zero_body(void *self, uint64 body, HostDeltaPageSize size) +{ +} + +/* host_delta_directory_tests_write writes no body, since no case takes one. */ +static void +host_delta_directory_tests_write(void *self, uint64 body, uint64 at, const uint8 *bytes, size_t bytes_len) +{ +} + +/* host_delta_directory_tests_set stores no body word, since no case takes one. */ +static void +host_delta_directory_tests_set(void *self, uint64 body, uint64 index, uint64 value) +{ +} + +/* host_delta_directory_tests_own lists no page, since no case takes one. */ +static void +host_delta_directory_tests_own(void *self, HostDeltaPagePage name) +{ +} + +/* host_delta_directory_tests_retiring is the epoch a retirement stamps: the current one. */ +static uint64 +host_delta_directory_tests_retiring(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->epoch; +} + +/* host_delta_directory_tests_retiring_now takes the epoch lock, which no other process holds. */ +static bool +host_delta_directory_tests_retiring_now(void *self, uint64 *result) +{ + *result = HOST_DELTA_DIRECTORY_TESTS_SELF(self)->epoch; + return true; +} + +/* host_delta_directory_tests_retired advances the epoch. */ +static void +host_delta_directory_tests_retired(void *self) +{ + HOST_DELTA_DIRECTORY_TESTS_SELF(self)->epoch++; +} + +/* host_delta_directory_tests_epoch is a test directory's current epoch. */ +static uint64 +host_delta_directory_tests_epoch(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->epoch; +} + +/* host_delta_directory_tests_oldest is a test directory's oldest epoch, which a reader holds below every retirement. */ +static uint64 +host_delta_directory_tests_oldest(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->oldest; +} + +/* host_delta_directory_tests_stop holds at no point. */ +static void +host_delta_directory_tests_stop(void *self, HostDeltaDirectoryPoint point) +{ +} + +/* host_delta_directory_tests_aged is a test directory's count of aged kinds: one. */ +static uint32 +host_delta_directory_tests_aged(void *self) +{ + return 1; +} + +/* host_delta_directory_tests_aged_kind is the number of a test directory's one aged kind. */ +static bool +host_delta_directory_tests_aged_kind(void *self, uint64 id, uint32 *result) +{ + *result = 0; + return id == HOST_DELTA_DIRECTORY_TESTS_AGED; +} + +/* host_delta_directory_tests_records is the count of a test directory's aged records, one for each slot. */ +static uint32 +host_delta_directory_tests_records(void *self, uint32 kind) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->slots; +} + +/* host_delta_directory_tests_record_word loads an aged record's word, counting it while a case counts. */ +static uint64 +host_delta_directory_tests_record_word(void *self, uint32 kind, uint32 index) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->passing) + d->visits++; + return d->records[index]; +} + +/* host_delta_directory_tests_record_swap stores an aged record's word over the one loaded. */ +static bool +host_delta_directory_tests_record_swap(void *self, uint32 kind, uint32 index, uint64 loaded, uint64 word) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->records[index] != loaded) + return false; + d->records[index] = word; + return true; +} + +/* host_delta_directory_tests_stamp loads an aged record's stamp. */ +static uint64 +host_delta_directory_tests_stamp(void *self, uint32 kind, uint32 index) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->stamps[index]; +} + +/* host_delta_directory_tests_set_stamp stores an aged record's stamp. */ +static void +host_delta_directory_tests_set_stamp(void *self, uint32 kind, uint32 index, uint64 stamp) +{ + HOST_DELTA_DIRECTORY_TESTS_SELF(self)->stamps[index] = stamp; +} + +/* host_delta_directory_tests_record_hand is an aged kind's hand, its first record. */ +static uint32 +host_delta_directory_tests_record_hand(void *self, uint32 kind) +{ + return 0; +} + +/* host_delta_directory_tests_record_nothing does nothing with an aged record, as its clear and listings do here. */ +static void +host_delta_directory_tests_record_nothing(void *self, uint32 kind, uint32 index) +{ +} + +/* host_delta_directory_tests_retired_head loads the retired list's head. */ +static uint64 +host_delta_directory_tests_retired_head(void *self) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->head; +} + +/* host_delta_directory_tests_retired_swap stores the retired list's head over the one loaded. */ +static bool +host_delta_directory_tests_retired_swap(void *self, uint64 loaded, uint64 word) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->head != loaded) + return false; + d->head = word; + return true; +} + +/* host_delta_directory_tests_link loads a slot's link. */ +static uint32 +host_delta_directory_tests_link(void *self, uint32 slot) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->links[slot]; +} + +/* host_delta_directory_tests_set_link stores a slot's link. */ +static void +host_delta_directory_tests_set_link(void *self, uint32 slot, uint32 value) +{ + HOST_DELTA_DIRECTORY_TESTS_SELF(self)->links[slot] = value; +} + +/* host_delta_directory_tests_record_retired_head loads the aged kind's retired list's head. */ +static uint64 +host_delta_directory_tests_record_retired_head(void *self, uint32 kind) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->record_head; +} + +/* host_delta_directory_tests_record_retired_swap stores the aged kind's retired list's head over the one loaded. */ +static bool +host_delta_directory_tests_record_retired_swap(void *self, uint32 kind, uint64 loaded, uint64 word) +{ + host_delta_directory_tests_Directory_t *d = HOST_DELTA_DIRECTORY_TESTS_SELF(self); + + if (d->record_head != loaded) + return false; + d->record_head = word; + return true; +} + +/* host_delta_directory_tests_record_link loads an aged record's link. */ +static uint64 +host_delta_directory_tests_record_link(void *self, uint32 kind, uint32 index) +{ + return HOST_DELTA_DIRECTORY_TESTS_SELF(self)->record_links[index]; +} + +/* host_delta_directory_tests_set_record_link stores an aged record's link. */ +static void +host_delta_directory_tests_set_record_link(void *self, uint32 kind, uint32 index, uint64 value) +{ + HOST_DELTA_DIRECTORY_TESTS_SELF(self)->record_links[index] = value; +} + +/* HOST_DELTA_DIRECTORY_TESTS_OPS is a test directory's Directory table. */ +static const HostDeltaDirectoryDirectoryOps HOST_DELTA_DIRECTORY_TESTS_OPS = { + .slots = host_delta_directory_tests_slots, + .budget = host_delta_directory_tests_budget, + .word = host_delta_directory_tests_word, + .swap = host_delta_directory_tests_swap, + .load = host_delta_directory_tests_load, + .store = host_delta_directory_tests_store, + .blocks = host_delta_directory_tests_blocks, + .region = host_delta_directory_tests_region, + .set_region = host_delta_directory_tests_set_region, + .block = host_delta_directory_tests_block, + .set_block = host_delta_directory_tests_set_block, + .lock_region = host_delta_directory_tests_nothing, + .unlock_region = host_delta_directory_tests_nothing, + .frees = host_delta_directory_tests_frees, + .freed = host_delta_directory_tests_freed, + .left = host_delta_directory_tests_left, + .leave = host_delta_directory_tests_leave, + .settle = host_delta_directory_tests_settle, + .kept = host_delta_directory_tests_kept, + .kept_swap = host_delta_directory_tests_kept_swap, + .waiting = host_delta_directory_tests_zero_count, + .count_waiter = host_delta_directory_tests_count_waiter, + .overshoot = host_delta_directory_tests_overshoot, + .overshoot_swap = host_delta_directory_tests_overshoot_swap, + .hand = host_delta_directory_tests_hand, + .hint = host_delta_directory_tests_hint, + .hint_set = host_delta_directory_tests_hint_slot, + .hint_clear = host_delta_directory_tests_hint_slot, + .spent = host_delta_directory_tests_spent, + .spend = host_delta_directory_tests_spend, + .lock = host_delta_directory_tests_lock, + .unlock = host_delta_directory_tests_nothing, + .zero = host_delta_directory_tests_zero_body, + .write = host_delta_directory_tests_write, + .set = host_delta_directory_tests_set, + .own = host_delta_directory_tests_own, + .disown = host_delta_directory_tests_own, + .retiring = host_delta_directory_tests_retiring, + .retiring_now = host_delta_directory_tests_retiring_now, + .retired = host_delta_directory_tests_retired, + .unretired = host_delta_directory_tests_nothing, + .epoch = host_delta_directory_tests_epoch, + .oldest = host_delta_directory_tests_oldest, + .stop = host_delta_directory_tests_stop, + .aged = host_delta_directory_tests_aged, + .aged_kind = host_delta_directory_tests_aged_kind, + .records = host_delta_directory_tests_records, + .record_word = host_delta_directory_tests_record_word, + .record_swap = host_delta_directory_tests_record_swap, + .stamp = host_delta_directory_tests_stamp, + .set_stamp = host_delta_directory_tests_set_stamp, + .record_hand = host_delta_directory_tests_record_hand, + .clear = host_delta_directory_tests_record_nothing, + .made = host_delta_directory_tests_record_nothing, + .unmade = host_delta_directory_tests_record_nothing, + .retired_head = host_delta_directory_tests_retired_head, + .retired_swap = host_delta_directory_tests_retired_swap, + .link = host_delta_directory_tests_link, + .set_link = host_delta_directory_tests_set_link, + .record_retired_head = host_delta_directory_tests_record_retired_head, + .record_retired_swap = host_delta_directory_tests_record_retired_swap, + .record_link = host_delta_directory_tests_record_link, + .set_record_link = host_delta_directory_tests_set_record_link, + .lock_pass = host_delta_directory_tests_nothing, + .unlock_pass = host_delta_directory_tests_nothing, +}; + +/* + * host_delta_directory_tests_retirements makes a test directory of a count of + * slots, every slot and aged record live, a reader holding the oldest epoch + * below every retirement, and retires ten pages and ten aged records, the + * last slots' and records'. + */ +static HostDeltaDirectoryDirectory +host_delta_directory_tests_retirements(uint32 slots) +{ + host_delta_directory_tests_Directory_t *d = &host_delta_directory_tests_directory; + HostDeltaDirectoryDirectory directory = {&HOST_DELTA_DIRECTORY_TESTS_OPS, d}; + HostDeltaPagePage names[10]; + HostDeltaLeaving leaving[10]; + uint64 owned; + uint64 live; + + memset(d, 0, sizeof(*d)); + d->slots = slots; + d->epoch = 5; + d->oldest = 1; + if (!host_delta_page_claim(0, &owned)) + std_panic(__FILE__, __LINE__, 0, "a free slot is claimed", sizeof("a free slot is claimed") - 1); + live = host_delta_page_link(owned); + for (uint32 at = 0; at < slots; at++) + { + d->words[at] = live; + d->records[at] = live; + } + for (uint32 i = 0; i < 10; i++) + { + uint32 at = slots - 1 - i; + + names[i] = host_delta_page_Page_of(at, live); + leaving[i].id = HOST_DELTA_DIRECTORY_TESTS_AGED; + leaving[i].location = host_delta_page_Page_value(host_delta_page_Page_of(at, live)); + } + host_delta_directory_retire(directory, leaving, lengthof(leaving), names, lengthof(names)); + return directory; +} + +/* host_delta_directory_tests_visited asserts that a case loaded want words, citing this file's line. */ +static void +host_delta_directory_tests_visited(uint32 line, uint64 want) +{ + uint64 visits = host_delta_directory_tests_directory.visits; + + if (visits != want) + std_assert_failed(__FILE__, line, 0, "==", &visits, std_u64_Debug_fmt, &want, std_u64_Debug_fmt, NULL); +} + +/* + * Bugs row 309's free pass: a pass after ten pages and ten aged records + * retire loads the words of those twenty and of no other slot or record, + * at a directory of 128 slots and at one of 4,096, so its work is the + * retirements and not the directory's size. + */ +void +test_host_delta_directory_tests_a_free_pass_loads_only_the_retirements(void) +{ + static const uint32 sizes[] = {128, HOST_DELTA_DIRECTORY_TESTS_MOST}; + + for (size_t i = 0; i < lengthof(sizes); i++) + { + HostDeltaDirectoryDirectory directory = host_delta_directory_tests_retirements(sizes[i]); + + host_delta_directory_tests_directory.passing = true; + host_delta_directory_pass(directory); + host_delta_directory_tests_visited(__LINE__, 20); + host_delta_directory_tests_directory.visits = 0; + host_delta_directory_pass(directory); + host_delta_directory_tests_visited(__LINE__, 20); + host_delta_directory_tests_directory.passing = false; + } +} + +/* + * Bugs row 309's refused allocation: an allocation the directory refuses, + * every slot live, runs a free pass that loads the words of the ten pages + * and ten aged records retired and of no other, at both sizes. + */ +void +test_host_delta_directory_tests_a_refused_allocations_pass_loads_only_the_retirements(void) +{ + static const uint32 sizes[] = {128, HOST_DELTA_DIRECTORY_TESTS_MOST}; + + for (size_t i = 0; i < lengthof(sizes); i++) + { + HostDeltaDirectoryDirectory directory = host_delta_directory_tests_retirements(sizes[i]); + HostDeltaPageSize wanted[1] = {{4096, 1}}; + StdVec names; + bool has_full; + uint64 full; + HostError error; + + std_vec_with_capacity_Checked(&names, 1, sizeof(HostDeltaPagePage), _Alignof(HostDeltaPagePage)); + host_delta_directory_tests_directory.passing = true; + if (!host_delta_directory_alloc(directory, NULL, wanted, lengthof(wanted), &names, &has_full, &full, &error)) + std_panic(__FILE__, __LINE__, 0, "a refused allocation answers its budget's signal", + sizeof("a refused allocation answers its budget's signal") - 1); + host_delta_directory_tests_directory.passing = false; + if (!has_full || full != 0) + std_panic(__FILE__, __LINE__, 0, "the allocation is refused", sizeof("the allocation is refused") - 1); + host_delta_directory_tests_visited(__LINE__, 20); + std_vec_free_Checked(&names, sizeof(HostDeltaPagePage), _Alignof(HostDeltaPagePage)); + } +} + #endif /* DOLTGRES_TEST_HOST */ diff --git a/src/doltgres/host/c/lib.c b/src/doltgres/host/c/lib.c index 1d8232410ae59..86c3d134b0541 100644 --- a/src/doltgres/host/c/lib.c +++ b/src/doltgres/host/c/lib.c @@ -382,6 +382,9 @@ host_WalKind_of(uint8 value, HostWalKind *result) case 13: *result = HOST_WAL_KIND_DATABASE_CHANGE; return true; + case 14: + *result = HOST_WAL_KIND_CARRY_UNLISTED; + return true; default: return false; } @@ -423,6 +426,8 @@ host_WalKind_value(HostWalKind self) return 12; case HOST_WAL_KIND_DATABASE_CHANGE: return 13; + case HOST_WAL_KIND_CARRY_UNLISTED: + return 14; } pg_unreachable(); } @@ -459,6 +464,8 @@ host_WalKind_name(HostWalKind self) return "MapCarried"; case HOST_WAL_KIND_DATABASE_CHANGE: return "DatabaseChange"; + case HOST_WAL_KIND_CARRY_UNLISTED: + return "CarryUnlisted"; #ifdef USE_INJECTION_POINTS case HOST_WAL_KIND_TEST: return "Test"; @@ -581,6 +588,17 @@ host_Xid_Debug_fmt(const void *value, StdFormatter *f) return std_DebugTuple_finish(&t); } +/* host_Command_Debug_fmt is Command's derived Debug. */ +bool +host_Command_Debug_fmt(const void *value, StdFormatter *f) +{ + const HostCommand *self = value; + StdDebugTuple t = std_fmt_debug_tuple(f, "Command", 7); + + std_DebugTuple_field(&t, &self->f0, std_u32_Debug_fmt); + return std_DebugTuple_finish(&t); +} + /* host_Row_PartialEq_eq reports whether two tuple ids are the same. */ bool host_Row_PartialEq_eq(const HostRow *self, const HostRow *other) @@ -1441,6 +1459,22 @@ host_wal_insert(HostIoIoDoors io, HostWalKind kind, const uint8 *body, size_t bo return host.ops->wal_insert(host.self, &raise, kind, body, body_len, result, error); } +/* host_NewCid_Debug_fmt is NewCid's derived Debug. */ +bool +host_NewCid_Debug_fmt(const void *value, StdFormatter *f) +{ + const HostNewCid *self = value; + StdOptionRef cmin = {self->has_cmin, {&self->cmin, host_Command_Debug_fmt}}; + StdOptionRef cmax = {self->has_cmax, {&self->cmax, host_Command_Debug_fmt}}; + StdOptionRef combo = {self->has_combo, {&self->combo, host_Command_Debug_fmt}}; + StdDebugStruct s = std_fmt_debug_struct(f, "NewCid", 6); + + std_DebugStruct_field(&s, "cmin", 4, &cmin, std_option_Debug_fmt); + std_DebugStruct_field(&s, "cmax", 4, &cmax, std_option_Debug_fmt); + std_DebugStruct_field(&s, "combo", 5, &combo, std_option_Debug_fmt); + return std_DebugStruct_finish(&s); +} + /* * host_wal_new_cid appends heap's own record of the command ids of a row a * transaction wrote or changed in a relation logical decoding reads, so @@ -2027,6 +2061,21 @@ host_refuse_recovery(HostIoIoDoors io, const char *detail, size_t detail_len) host.ops->refuse_recovery(host.self, detail, detail_len); } +/* + * host_load_pauses loads pauses a later checkpoint window runs, so a failure + * to load one comes before the change the window finishes, as Postgres' + * INJECTION_POINT_LOAD comes before its START_CRIT_SECTION; with no pauses + * built in, it does nothing. + */ +bool +host_load_pauses(HostIoIoDoors io, const char *const *pauses, size_t pauses_len, HostError *error) +{ + HostHost host = host_current(); + HostMayRaise raise = host_doors_of_io_doors(io); + + return host.ops->load_pauses(host.self, &raise, pauses, pauses_len, error); +} + /* * host_delay_checkpoint opens a window holding off the start of a checkpoint * until it finishes, with the names of the pauses inside it, which its open @@ -2189,30 +2238,31 @@ host_collection_end(void) /* * host_collection_ask asks the collector worker for a full collection of a - * database's repository and waits for its answer, the packs it retired, those + * database's repository, emptying a tablespace unless that is + * CHUNKS_PACKS_DEFAULT_SPACE, and waits for its answer, the packs it retired, those * it copied out, and its live bytes, each PG_UINT64_MAX when it failed (the * collector text's §8.2); a cancel ends the wait with Postgres' error, the ask * taken back, and the collection goes on. */ bool -host_collection_ask(HostIoIo *io, uint32 database, uint64 *result, HostError *error) +host_collection_ask(HostIoIo *io, uint32 database, uint32 emptied, uint64 *result, HostError *error) { HostHost host = host_current(); HostMayRaise raise = host_doors(HOST_IO_IO_READ_DOORS(io)); - return host.ops->collection_ask(host.self, &raise, database, result, error); + return host.ops->collection_ask(host.self, &raise, database, emptied, result, error); } /* - * host_collection_asked returns the unanswered ask's word and the database it - * asks for, which the collector worker reads. + * host_collection_asked returns the unanswered ask's word, the database it + * asks for, and the tablespace it empties, which the collector worker reads. */ bool -host_collection_asked(uint64 *result_0, uint32 *result_1) +host_collection_asked(uint64 *result_0, uint32 *result_1, uint32 *result_2) { HostHost host = host_current(); - return host.ops->collection_asked(host.self, result_0, result_1); + return host.ops->collection_asked(host.self, result_0, result_1, result_2); } /* host_collection_answer answers an ask, by its word, with the collection's counts, and wakes its asker. */ @@ -2233,6 +2283,15 @@ host_collector_started(void) host.ops->collector_started(host.self); } +/* host_collector_running reports whether a collector worker has started, without which an ask would wait for good. */ +bool +host_collector_running(void) +{ + HostHost host = host_current(); + + return host.ops->collector_running(host.self); +} + /* * host_database_locked takes a database's lock in session mode without * waiting, answering whether it took it, so the collector worker opens the @@ -2742,6 +2801,19 @@ host_tests_Test_Host_refuse_recovery(void *self, const char *detail, size_t deta std_panic(RUST_FILE, 2157, 13, text, sizeof(text) - 1); } +/* host_tests_Test_Host_load_pauses loads nothing: the test host runs no pause. */ +static bool +host_tests_Test_Host_load_pauses(void *self, const HostMayRaise *raise, const char *const *pauses, size_t pauses_len, + HostError *error) +{ + (void) self; + (void) raise; + (void) pauses; + (void) pauses_len; + (void) error; + return true; +} + /* host_tests_Test_Host_delay_checkpoints is unreachable here: only `pghost` makes the capability a WAL seam takes. */ static bool host_tests_Test_Host_delay_checkpoints(void *self, const HostMayRaise *raise, const char *const *pauses, size_t pauses_len, HostError *error) @@ -3646,7 +3718,8 @@ host_tests_Test_Host_collection_end(void *self) /* host_tests_Test_Host_collection_ask is unreachable here: no test of `host` collects. */ static bool -host_tests_Test_Host_collection_ask(void *self, const HostMayRaise *raise, uint32 database, uint64 *result, HostError *error) +host_tests_Test_Host_collection_ask(void *self, const HostMayRaise *raise, uint32 database, uint32 emptied, + uint64 *result, HostError *error) { static const char text[] = "internal error: entered unreachable code: no test of `host` collects"; @@ -3655,7 +3728,7 @@ host_tests_Test_Host_collection_ask(void *self, const HostMayRaise *raise, uint3 /* host_tests_Test_Host_collection_asked is unreachable here: no test of `host` collects. */ static bool -host_tests_Test_Host_collection_asked(void *self, uint64 *result_0, uint32 *result_1) +host_tests_Test_Host_collection_asked(void *self, uint64 *result_0, uint32 *result_1, uint32 *result_2) { static const char text[] = "internal error: entered unreachable code: no test of `host` collects"; @@ -3680,6 +3753,15 @@ host_tests_Test_Host_collector_started(void *self) std_panic(RUST_FILE, 2590, 13, text, sizeof(text) - 1); } +/* host_tests_Test_Host_collector_running is unreachable here: no test of `host` collects. */ +static bool +host_tests_Test_Host_collector_running(void *self) +{ + static const char text[] = "internal error: entered unreachable code: no test of `host` collects"; + + std_panic(__FILE__, __LINE__, 0, text, sizeof(text) - 1); +} + /* host_tests_Test_Host_database_locked is unreachable here: no test of `host` collects. */ static bool host_tests_Test_Host_database_locked(void *self, const HostMayRaise *raise, uint32 database, bool *result, HostError *error) @@ -3889,6 +3971,7 @@ static const HostHostOps HOST_TESTS_TEST_HOST_OPS = { .log = host_tests_Test_Host_log, .refuse_recovery = host_tests_Test_Host_refuse_recovery, .delay_checkpoints = host_tests_Test_Host_delay_checkpoints, + .load_pauses = host_tests_Test_Host_load_pauses, .resume_checkpoints = host_tests_Test_Host_resume_checkpoints, .xid_status = host_tests_Test_Host_xid_status, .xid_in_progress = host_tests_Test_Host_xid_in_progress, @@ -3995,6 +4078,7 @@ static const HostHostOps HOST_TESTS_TEST_HOST_OPS = { .collection_asked = host_tests_Test_Host_collection_asked, .collection_answer = host_tests_Test_Host_collection_answer, .collector_started = host_tests_Test_Host_collector_started, + .collector_running = host_tests_Test_Host_collector_running, .database_locked = host_tests_Test_Host_database_locked, .database_unlocked = host_tests_Test_Host_database_unlocked, .database_lock_waited = host_tests_Test_Host_database_lock_waited, diff --git a/src/doltgres/host/c/shared.c b/src/doltgres/host/c/shared.c index 3c376f66c5406..00c874857d0bd 100644 --- a/src/doltgres/host/c/shared.c +++ b/src/doltgres/host/c/shared.c @@ -109,8 +109,9 @@ host_shared_Kind_pool(const HostSharedPoolOps *s) /* * host_shared_Kind_aged returns the line of a pool the delta's epochs free, - * whose records hold one atomic past their shape's, the host's, for the - * epoch of their retirement. + * whose records hold two atomics past their shape's, the host's: the epoch of + * their retirement, and their link in their kind's retired list (bugs row + * 309). */ HostSharedKind host_shared_Kind_aged(const HostSharedAgedOps *s) @@ -122,7 +123,7 @@ host_shared_Kind_aged(const HostSharedAgedOps *s) form.u.aged.names = s->names; form.u.aged.names_len = s->names_len; kind = host_shared_Kind_of(&s->shape, form); - kind.atomics = STD_ADD(usize, kind.atomics, 1, 221, 9); + kind.atomics = STD_ADD(usize, kind.atomics, 2, 221, 9); return kind; } diff --git a/src/doltgres/pg/c/lib.c b/src/doltgres/pg/c/lib.c index e4daaf2ca0b23..e4ca967d517f0 100644 --- a/src/doltgres/pg/c/lib.c +++ b/src/doltgres/pg/c/lib.c @@ -212,16 +212,13 @@ pg_snapshot_view_closure_0(void *captures, const uint32 *xids, size_t count, con } } -/* pg_snapshot_view reads a snapshot an entry got into its view, its arrays lent for as long as the snapshot is. */ -bool -pg_snapshot_view(const PgMayRaise *raise, const PgSnapshot *snapshot, PgSnapshotView *result, PgPgError *error) +/* pg_snapshot_view_of reads a snapshot's fields into its view, its arrays lent for as long as the snapshot is. */ +static void +pg_snapshot_view_of(DgSnapshotView fields, PgSnapshotView *result) { - DgSnapshotView fields; size_t subxcnt; StdTryFromIntError tried; - if (!pg_calls_snapshot_fields(raise, snapshot, &fields, error)) - return false; result->kind = fields.kind; result->xmin = fields.xmin; result->xmax = fields.xmax; @@ -232,6 +229,33 @@ pg_snapshot_view(const PgMayRaise *raise, const PgSnapshot *snapshot, PgSnapshot if (!std_usize_try_from_i32(fields.subxcnt, &subxcnt, &tried)) subxcnt = 0; pg_snapshot_view_closure_0(NULL, fields.subxip, subxcnt, &result->subxip, &result->subxip_len); +} + +/* + * pg_transaction_snapshot_view reads the transaction's snapshot, as + * GetTransactionSnapshot takes it, into its view, its arrays lent until the + * backend's next snapshot. + */ +bool +pg_transaction_snapshot_view(const PgMayRaise *raise, PgSnapshotView *result, PgPgError *error) +{ + DgSnapshotView fields; + + if (!pg_calls_transaction_snapshot_fields(raise, &fields, error)) + return false; + pg_snapshot_view_of(fields, result); + return true; +} + +/* pg_snapshot_view reads a snapshot an entry got into its view, its arrays lent for as long as the snapshot is. */ +bool +pg_snapshot_view(const PgMayRaise *raise, const PgSnapshot *snapshot, PgSnapshotView *result, PgPgError *error) +{ + DgSnapshotView fields; + + if (!pg_calls_snapshot_fields(raise, snapshot, &fields, error)) + return false; + pg_snapshot_view_of(fields, result); return true; } diff --git a/src/doltgres/pg/calls.list b/src/doltgres/pg/calls.list index fa61bc519d2d8..5c94baf7f80cd 100644 --- a/src/doltgres/pg/calls.list +++ b/src/doltgres/pg/calls.list @@ -254,6 +254,7 @@ log_new_cid uint64 DgStorage storage, uint64 tid, CommandId cmin, CommandId cmax rel_att_type Oid Relation rel, int attnum noraise nocatalog it reads the relation's tuple descriptor, which its relcache entry holds TupleDescAttr(RelationGetDescr(rel), attnum - 1)->atttypid test/t/033_scan.pl: a scan of a table keyed by later columns gives its rows in key order register_subxact_end void raises nocatalog RegisterSubXactCallback allocates its entry in TopMemoryContext, which raises only when memory runs out; it reads no catalog RegisterSubXactCallback(doltgres_subxact_ended, NULL) test/t/044_removal.pl: a rolled-back savepoint's rows are taken back and its parent's kept snapshot_fields DgSnapshotView Snapshot snapshot raises nocatalog ReadNextFullTransactionId takes XidGenLock shared to widen an MVCC or historic snapshot's bounds against the next full xid, which every bound is within 2^31 of, and FullTransactionIdFromAllowableAt keeps a special xid as it is; the rest reads the snapshot's own fields, whose arrays live as long as the snapshot, and of any other kind only its type, which alone its rule reads, as heap's do, since a Dirty snapshot's bounds are unset until a read sets them; it reads no catalog { if (snapshot->snapshot_type != SNAPSHOT_MVCC && snapshot->snapshot_type != SNAPSHOT_HISTORIC_MVCC) *out = (DgSnapshotView) {(int) snapshot->snapshot_type}; else { FullTransactionId next = ReadNextFullTransactionId(); *out = (DgSnapshotView) {(int) snapshot->snapshot_type, U64FromFullTransactionId(FullTransactionIdFromAllowableAt(next, snapshot->xmin)), U64FromFullTransactionId(FullTransactionIdFromAllowableAt(next, snapshot->xmax)), snapshot->curcid, snapshot->takenDuringRecovery, snapshot->suboverflowed, snapshot->xip, snapshot->xcnt, snapshot->subxip, snapshot->subxcnt}; } } test/t/042_snapshot.pl: a snapshot taken while a transaction runs sees it in progress after it commits +transaction_snapshot_fields DgSnapshotView raises nocatalog GetTransactionSnapshot reads the proc array, as heap's concurrent build scan takes its snapshot, and ReadNextFullTransactionId takes XidGenLock shared to widen the snapshot's bounds against the next full xid, as snapshot_fields does; its arrays live until the backend's next snapshot, and it reads no catalog { Snapshot snapshot = GetTransactionSnapshot(); FullTransactionId next = ReadNextFullTransactionId(); *out = (DgSnapshotView) {(int) snapshot->snapshot_type, U64FromFullTransactionId(FullTransactionIdFromAllowableAt(next, snapshot->xmin)), U64FromFullTransactionId(FullTransactionIdFromAllowableAt(next, snapshot->xmax)), snapshot->curcid, snapshot->takenDuringRecovery, snapshot->suboverflowed, snapshot->xip, snapshot->xcnt, snapshot->subxip, snapshot->subxcnt}; } test/t/045_btree.pl: a REINDEX CONCURRENTLY computes an indexed row's expression once, as heap's validation does snapshot_removable bool Snapshot snapshot, uint64 xid raises nocatalog GlobalVisTestIsRemovableFullXid may take ProcArrayLock to move the NonVacuumable snapshot's own horizon, which raises only when the process holds too many locks; it reads no catalog *out = GlobalVisTestIsRemovableFullXid(snapshot->vistest, FullTransactionIdFromU64(xid), true) test/t/042_snapshot.pl: a NonVacuumable snapshot sees a deleted row until its deleter passes the horizon scan_relation Relation const TableScanDesc scan noraise nocatalog it reads the scan's own field, the relation its begin was given scan->rs_rd test/t/042_snapshot.pl: a seq scan gives the rows its snapshot sees of the base and the delta scan_snapshot Snapshot const TableScanDesc scan noraise nocatalog it reads the scan's own field, the snapshot its begin was given scan->rs_snapshot test/t/042_snapshot.pl: a seq scan gives the rows its snapshot sees of the base and the delta @@ -292,7 +293,7 @@ lock_tuple bool Oid dbid, Oid relid, uint64 tid, int mode, bool wait raises noca unlock_tuple void Oid dbid, Oid relid, uint64 tid, int mode escalates nocatalog LockRelease warns only for a lock this backend doesn't hold, which the tuple lock's guard, its only caller, rules out by construction { static const LOCKMODE modes[] = {AccessShareLock, RowShareLock, ExclusiveLock, AccessExclusiveLock}; LOCKTAG tag; SET_LOCKTAG_TUPLE(tag, dbid, relid, (BlockNumber) (tid >> 16), (OffsetNumber) (tid & 0xffff)); LockRelease(&tag, modes[mode], false); } test/t/046_update.pl: a second updater of a row waits for the first's transaction rel_key_immediate bool Relation rel raises catalog it reads the relation's primary key index's pg_index row through one INDEXRELID lookup, refusing a missing row with relcache.c's words { Oid index = RelationGetPrimaryKeyIndex(rel, true); *out = true; if (OidIsValid(index)) { HeapTuple t = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index)); if (!HeapTupleIsValid(t)) elog(ERROR, "cache lookup failed for index %u", index); *out = ((Form_pg_index) GETSTRUCT(t))->indimmediate; ReleaseSysCache(t); } } test/t/046_update.pl: a deferrable key's rows go beside a live row rel_key_equal_image bool Relation rel raises catalog _bt_allequalimage calls each key column's opclass's BTEQUALIMAGE_PROC, which may run user code, and opens the primary key index { Oid index = RelationGetPrimaryKeyIndex(rel, true); *out = true; if (OidIsValid(index)) { Relation irel = index_open(index, AccessShareLock); *out = _bt_allequalimage(irel, false); index_close(irel, AccessShareLock); } } test/t/046_update.pl: two writers of a text key under a nondeterministic collation, one waiting -refuse_duplicate_key void Relation rel, TupleTableSlot *slot raises catalog it raises the btree's unique violation, 23505, with the primary key index's name and the key's values as BuildIndexValueDescription gives them, which run the columns' output functions { Oid index = RelationGetPrimaryKeyIndex(rel, true); Relation irel = index_open(index, AccessShareLock); IndexInfo *info = BuildIndexInfo(irel); Datum values[INDEX_MAX_KEYS]; bool isnull[INDEX_MAX_KEYS]; char *desc; FormIndexDatum(info, slot, NULL, values, isnull); desc = BuildIndexValueDescription(irel, values, isnull); ereport(ERROR, (errcode(ERRCODE_UNIQUE_VIOLATION), errmsg("duplicate key value violates unique constraint \"%s\"", RelationGetRelationName(irel)), desc ? errdetail("Key %s already exists.", desc) : 0, errtableconstraint(rel, RelationGetRelationName(irel)))); } test/t/046_update.pl: an insert of a live row's key is refused as heap's is +refuse_duplicate_key void Relation rel, TupleTableSlot *slot raises catalog it raises the btree's unique violation, 23505, with the primary key index's name and the key's values as BuildIndexValueDescription gives them, which run the columns' output functions, or, for a primary key WITHOUT OVERLAPS, whose index is an exclusion constraint's, the exclusion violation, 23P01, worded as check_exclusion_or_unique_constraint words it, the existing key the same values { Oid index = RelationGetPrimaryKeyIndex(rel, true); Relation irel = index_open(index, AccessShareLock); IndexInfo *info = BuildIndexInfo(irel); Datum values[INDEX_MAX_KEYS]; bool isnull[INDEX_MAX_KEYS]; char *desc; FormIndexDatum(info, slot, NULL, values, isnull); desc = BuildIndexValueDescription(irel, values, isnull); if (irel->rd_index->indisexclusion) ereport(ERROR, (errcode(ERRCODE_EXCLUSION_VIOLATION), errmsg("conflicting key value violates exclusion constraint \"%s\"", RelationGetRelationName(irel)), desc ? errdetail("Key %s conflicts with existing key %s.", desc, desc) : errdetail("Key conflicts with existing key."), errtableconstraint(rel, RelationGetRelationName(irel)))); ereport(ERROR, (errcode(ERRCODE_UNIQUE_VIOLATION), errmsg("duplicate key value violates unique constraint \"%s\"", RelationGetRelationName(irel)), desc ? errdetail("Key %s already exists.", desc) : 0, errtableconstraint(rel, RelationGetRelationName(irel)))); } test/t/046_update.pl: an insert of a live row's key is refused as heap's is; test/t/046_update.pl: an insert or an update of a WITHOUT OVERLAPS primary key's existing key is refused as heap's exclusion check refuses it rel_key_index Oid Relation rel raises catalog RelationGetPrimaryKeyIndex reads the relation cache's index list, which it may build from the catalog *out = RelationGetPrimaryKeyIndex(rel, true) test/t/046_update.pl: an insert of a live row's key is refused as heap's is combo_command uint32 uint64 xid, uint32 cmin, uint32 cmax raises nocatalog HeapTupleHeaderAdjustCmax gives a change of a row its own transaction wrote a combo command id from the backend's combo table, which GetComboCommandId grows in TopTransactionContext and which can raise out of memory; the header it reads holds only the row's writer and command, so no catalog is read { HeapTupleHeaderData header = {0}; bool combo; HeapTupleHeaderSetXmin(&header, XidFromFullTransactionId(FullTransactionIdFromU64(xid))); HeapTupleHeaderSetCmin(&header, cmin); *out = cmax; HeapTupleHeaderAdjustCmax(&header, out, &combo); } test/t/046_update.pl: a row its own transaction inserted and then changed shows the combo command id as heap's does set_failure void TM_FailureData *tmfd, uint64 ctid, uint32 xmax, uint32 cmax noraise nocatalog it writes the failure data's own fields, as heap_delete, heap_update, and heap_lock_tuple fill them ItemPointerSet(&tmfd->ctid, (BlockNumber) (ctid >> 16), (OffsetNumber) (ctid & 0xffff)); tmfd->xmax = xmax; tmfd->cmax = cmax test/t/046_update.pl: a second updater of a row waits for the first's transaction @@ -398,7 +399,7 @@ start_fold_worker uint64 Oid database, bool asked raises nocatalog RegisterDynam fold_worker_running bool uint64 handle raises nocatalog GetBackgroundWorkerPid reads the worker's slot under BackgroundWorkerLock, raising only when the process holds too many locks { pid_t pid; *out = GetBackgroundWorkerPid((BackgroundWorkerHandle *) (uintptr_t) handle, &pid) != BGWH_STOPPED; } test/t/062_fold_trigger.pl: a storage whose delta passes fold_threshold is folded with no call from a test free_worker_handle void uint64 handle noraise nocatalog pfree frees the handle start_fold_worker allocated, which holds nothing else pfree((void *) (uintptr_t) handle) test/t/062_fold_trigger.pl: a storage whose delta passes fold_threshold is folded with no call from a test worker_connect_database void Oid database raises catalog BackgroundWorkerInitializeConnectionByOid connects the worker to its database, reading pg_database, after the worker takes SIGHUP as a reload of its configuration { pqsignal(SIGHUP, SignalHandlerForConfigReload); BackgroundWorkerUnblockSignals(); BackgroundWorkerInitializeConnectionByOid(database, InvalidOid, 0); } test/t/062_fold_trigger.pl: a storage whose delta passes fold_threshold is folded with no call from a test -fold_as_owner bool Oid relid, Oid relfilenode raises catalog in a transaction of its own it opens the relation, an index's table before it, as Postgres locks a table before its indexes, which reads its catalog, and passes over one gone or rewritten since, or one whose lock another transaction holds, as a transaction that made or truncated the table holds it while its writes wait at delta_memory for the folds of the others; it runs the fold as the relation's owner in a security-restricted operation with search_path restricted and every user-settable setting at its built-in default (PLAN item 6's User code in folds), lc_messages at C because its built-in empty value, the environment's locale, is one only the server's start may set, set at a GUC nest level the commit undoes, as VACUUM runs index functions { Relation rel = NULL; Relation heap = NULL; Oid heapid; Oid saved_user; int saved_context; StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); heapid = IndexGetRelation(relid, true); if (OidIsValid(heapid) && ConditionalLockRelationOid(heapid, AccessShareLock) && (heap = try_relation_open(heapid, NoLock)) == NULL) UnlockRelationOid(heapid, AccessShareLock); if ((!OidIsValid(heapid) || heap != NULL) && ConditionalLockRelationOid(relid, AccessShareLock) && (rel = try_relation_open(relid, NoLock)) == NULL) UnlockRelationOid(relid, AccessShareLock); *out = false; if (rel != NULL && rel->rd_locator.relNumber == relfilenode && (rel->rd_index == NULL || heap != NULL)) { int level; GetUserIdAndSecContext(&saved_user, &saved_context); SetUserIdAndSecContext(rel->rd_rel->relowner, saved_context | SECURITY_RESTRICTED_OPERATION); level = NewGUCNestLevel(); { int count; struct config_generic **vars = get_guc_variables(&count); for (int at = 0; at < count; at++) { struct config_generic *var = vars[at]; const char *value = NULL; char number[64]; if ((var->context != PGC_SUSET && var->context != PGC_USERSET) || (var->flags & GUC_NO_RESET_ALL)) continue; switch (var->vartype) { case PGC_BOOL: value = var->_bool.boot_val ? "on" : "off"; break; case PGC_INT: snprintf(number, sizeof(number), "%d", var->_int.boot_val); value = number; break; case PGC_REAL: snprintf(number, sizeof(number), "%.17g", var->_real.boot_val); value = number; break; case PGC_STRING: value = strcmp(var->name, "lc_messages") == 0 ? "C" : var->_string.boot_val; break; case PGC_ENUM: value = config_enum_lookup_by_value(var, var->_enum.boot_val); break; } if (value != NULL) (void) set_config_option(var->name, value, PGC_SUSET, PGC_S_SESSION, GUC_ACTION_SAVE, true, ERROR, false); } } RestrictSearchPath(); *out = doltgres_fold_relation(rel, heap != NULL ? heap : rel); AtEOXact_GUC(false, level); SetUserIdAndSecContext(saved_user, saved_context); } if (rel != NULL) relation_close(rel, AccessShareLock); if (heap != NULL) relation_close(heap, AccessShareLock); PopActiveSnapshot(); CommitTransactionCommand(); } test/t/062_fold_trigger.pl: a fold orders a user comparator's keys under the built-in settings, not the server's +fold_as_owner bool Oid relid, Oid relfilenode raises catalog in a transaction of its own it opens the relation, an index's table before it, as Postgres locks a table before its indexes, which reads its catalog, and passes over one gone or rewritten since, or one whose lock another transaction holds, as a transaction that made or truncated the table holds it while its writes wait at delta_memory for the folds of the others; it runs the fold as the relation's owner in a security-restricted operation with search_path restricted and every user-settable setting at its built-in default (PLAN item 6's User code in folds), lc_messages at C because its built-in empty value, the environment's locale, is one only the server's start may set, set at a GUC nest level the commit undoes, as VACUUM runs index functions; before its snapshot it marks its transaction as a safe concurrent index build marks its own (PROC_IN_SAFE_IC, which only WaitForOlderSnapshots reads and its commit clears), so a concurrent index build doesn't wait for it: the fold reads its storage's base and delta only by the fold horizon, never by its snapshot or through an index, the horizon stays at or below every running snapshot's xmin, the build's own included, and the fold only applies committed versions to a new base, adding no row and no index entry, so neither a fold of the build's table nor one of the new index's own storage leaves out an entry the index must hold, and the only user code it runs is the key's support and collation functions, which a safe build also runs { Relation rel = NULL; Relation heap = NULL; Oid heapid; Oid saved_user; int saved_context; StartTransactionCommand(); LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); MyProc->statusFlags |= PROC_IN_SAFE_IC; ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags; LWLockRelease(ProcArrayLock); PushActiveSnapshot(GetTransactionSnapshot()); heapid = IndexGetRelation(relid, true); if (OidIsValid(heapid) && ConditionalLockRelationOid(heapid, AccessShareLock) && (heap = try_relation_open(heapid, NoLock)) == NULL) UnlockRelationOid(heapid, AccessShareLock); if ((!OidIsValid(heapid) || heap != NULL) && ConditionalLockRelationOid(relid, AccessShareLock) && (rel = try_relation_open(relid, NoLock)) == NULL) UnlockRelationOid(relid, AccessShareLock); *out = false; if (rel != NULL && rel->rd_locator.relNumber == relfilenode && (rel->rd_index == NULL || heap != NULL)) { int level; GetUserIdAndSecContext(&saved_user, &saved_context); SetUserIdAndSecContext(rel->rd_rel->relowner, saved_context | SECURITY_RESTRICTED_OPERATION); level = NewGUCNestLevel(); { int count; struct config_generic **vars = get_guc_variables(&count); for (int at = 0; at < count; at++) { struct config_generic *var = vars[at]; const char *value = NULL; char number[64]; if ((var->context != PGC_SUSET && var->context != PGC_USERSET) || (var->flags & GUC_NO_RESET_ALL)) continue; switch (var->vartype) { case PGC_BOOL: value = var->_bool.boot_val ? "on" : "off"; break; case PGC_INT: snprintf(number, sizeof(number), "%d", var->_int.boot_val); value = number; break; case PGC_REAL: snprintf(number, sizeof(number), "%.17g", var->_real.boot_val); value = number; break; case PGC_STRING: value = strcmp(var->name, "lc_messages") == 0 ? "C" : var->_string.boot_val; break; case PGC_ENUM: value = config_enum_lookup_by_value(var, var->_enum.boot_val); break; } if (value != NULL) (void) set_config_option(var->name, value, PGC_SUSET, PGC_S_SESSION, GUC_ACTION_SAVE, true, ERROR, false); } } RestrictSearchPath(); *out = doltgres_fold_relation(rel, heap != NULL ? heap : rel); AtEOXact_GUC(false, level); SetUserIdAndSecContext(saved_user, saved_context); } if (rel != NULL) relation_close(rel, AccessShareLock); if (heap != NULL) relation_close(heap, AccessShareLock); PopActiveSnapshot(); CommitTransactionCommand(); } test/t/062_fold_trigger.pl: a fold orders a user comparator's keys under the built-in settings, not the server's worker_connect void raises catalog BackgroundWorkerInitializeConnection connects the worker to no database, for its reads of the shared catalogs, after the worker takes SIGHUP as a reload of its configuration { pqsignal(SIGHUP, SignalHandlerForConfigReload); BackgroundWorkerUnblockSignals(); BackgroundWorkerInitializeConnection(NULL, NULL, 0); } test/t/056_collect.pl: the collector worker collects every database's repository on its own latch_sleep void int64 ms raises nocatalog WaitLatch waits on the process's latch, exiting with the postmaster, and CHECK_FOR_INTERRUPTS raises a pending cancel or ends a worker at its SIGTERM; it reads no catalog { (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, ms, PG_WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } test/t/056_collect.pl: the collector worker collects every database's repository on its own disk_space void const char *path, DgBuffer sizes raises nocatalog statvfs or GetDiskFreeSpaceExA reads the volume's sizes, raising when the system can't say; it reads no catalog { uint64 found[2]; doltgres_disk_space(path, &found[0], &found[1]); memcpy(sizes.data, found, Min(sizes.len, sizeof(found))); } test/t/056_collect.pl: the data directory's volume reports free bytes below its size @@ -430,7 +431,7 @@ filelocator_storage DgStorage const RelFileLocator *locator noraise nocatalog it drop_storage void Relation rel raises nocatalog RelationDropStorage schedules the relation's file's unlink at commit, raising only when memory runs out; it reads no catalog RelationDropStorage(rel) test/t/047_rewrite.pl: SET TABLESPACE keeps every row version with its ctid, xmin, xmax, cmin, and cmax, for an older snapshot too xlog_is_needed bool noraise nocatalog XLogIsNeeded reads the wal_level setting XLogIsNeeded() test/t/047_rewrite.pl: SET TABLESPACE keeps every row version with its ctid, xmin, xmax, cmin, and cmax, for an older snapshot too index_routine_forks bool Relation rel noraise nocatalog it compares the relcache entry's routine's aminsert with the fork's rel->rd_indam->aminsert == doltgres_index_insert test/t/047_rewrite.pl: ALTER INDEX SET TABLESPACE keeps a fork index's entries, for an index of a keyless table too -fold_horizon uint64 Relation rel raises nocatalog GetOldestNonRemovableTransactionId takes ProcArrayLock shared and reads the caller's database's horizon, which the Assert holds to the relation's; LWLockAcquire then takes it shared again to lower that horizon to the xmin and xid of each process in VACUUM the horizon covers, which ComputeXidHorizons passes over though a fold takes no lock that conflicts with VACUUM's, read through the PGPROC mirror of their flags, which ProcArrayEndTransaction clears at a transaction's end and InitProcess at a slot's reuse; it raises only past the held locks' limit { TransactionId horizon = GetOldestNonRemovableTransactionId(rel); bool shared = rel->rd_rel->relisshared; Assert(shared || rel->rd_locator.dbOid == MyDatabaseId || !OidIsValid(MyDatabaseId) || RecoveryInProgress()); LWLockAcquire(ProcArrayLock, LW_SHARED); for (int i = 0; i < ProcGlobal->allProcCount; i++) { PGPROC *proc = &ProcGlobal->allProcs[i]; uint8 flags = *(volatile uint8 *) &proc->statusFlags; TransactionId xmin = *(volatile TransactionId *) &proc->xmin; TransactionId xid = *(volatile TransactionId *) &proc->xid; if (!(flags & PROC_IN_VACUUM) || (!shared && proc->databaseId != rel->rd_locator.dbOid)) continue; if (TransactionIdIsNormal(xmin) && TransactionIdPrecedes(xmin, horizon)) horizon = xmin; if (TransactionIdIsNormal(xid) && TransactionIdPrecedes(xid, horizon)) horizon = xid; } LWLockRelease(ProcArrayLock); *out = U64FromFullTransactionId(FullTransactionIdFromAllowableAt(ReadNextFullTransactionId(), horizon)); } test/t/050_fold.pl: a lazy VACUUM held paused holds back a fold of its database's table, and finishes after it +fold_horizon uint64 Relation rel raises nocatalog GetOldestNonRemovableTransactionId takes ProcArrayLock shared and reads the caller's database's horizon, which the Assert holds to the relation's; LWLockAcquire then takes it shared again to lower that horizon to the xmin and xid of each process in VACUUM the horizon covers, which ComputeXidHorizons passes over though a fold takes no lock that conflicts with VACUUM's, read through the PGPROC mirror of their flags, which ProcArrayEndTransaction clears at a transaction's end and InitProcess at a slot's reuse; it raises only past the held locks' limit { TransactionId horizon = GetOldestNonRemovableTransactionId(rel); TransactionId own = MyProc->xmin; bool shared = rel->rd_rel->relisshared; Assert(shared || rel->rd_locator.dbOid == MyDatabaseId || !OidIsValid(MyDatabaseId) || RecoveryInProgress()); if (TransactionIdIsNormal(own) && TransactionIdPrecedes(own, horizon)) horizon = own; LWLockAcquire(ProcArrayLock, LW_SHARED); for (int i = 0; i < ProcGlobal->allProcCount; i++) { PGPROC *proc = &ProcGlobal->allProcs[i]; uint8 flags = *(volatile uint8 *) &proc->statusFlags; TransactionId xmin = *(volatile TransactionId *) &proc->xmin; TransactionId xid = *(volatile TransactionId *) &proc->xid; if (!(flags & PROC_IN_VACUUM) || (!shared && proc->databaseId != rel->rd_locator.dbOid)) continue; if (TransactionIdIsNormal(xmin) && TransactionIdPrecedes(xmin, horizon)) horizon = xmin; if (TransactionIdIsNormal(xid) && TransactionIdPrecedes(xid, horizon)) horizon = xid; } LWLockRelease(ProcArrayLock); *out = U64FromFullTransactionId(FullTransactionIdFromAllowableAt(ReadNextFullTransactionId(), horizon)); } test/t/050_fold.pl: a lazy VACUUM held paused holds back a fold of its database's table, and finishes after it lwlock_try_sure bool LWLock *lock, LWLockMode mode escalates nocatalog LWLockConditionalAcquire raises only when too many locks are held, which its one caller rules out: a free pass takes the epoch lock behind an Io no guard has borrowed, so with no lock of the fork's held, and anything raised becomes a PANIC; it never waits and reads no catalog LWLockConditionalAcquire(lock, mode) test/t/017_pages.pl: a pass that loaded its oldest epoch before a record's retirement walks while it is held atomic_or_u64 uint64 pg_atomic_uint64 *atomic, uint64 bits noraise nocatalog pg_atomic_fetch_or_u64 only sets bits pg_atomic_fetch_or_u64(atomic, bits) test/t/017_pages.pl: bytes appended by one backend are read by another atomic_and_u64 uint64 pg_atomic_uint64 *atomic, uint64 bits noraise nocatalog pg_atomic_fetch_and_u64 only clears bits pg_atomic_fetch_and_u64(atomic, bits) test/t/017_pages.pl: bytes appended by one backend are read by another @@ -502,7 +503,7 @@ rel_local bool Relation rel noraise nocatalog it reads the relcache entry's own warn_parallel_temporary void Relation rel raises nocatalog ereport at WARNING raises only when memory for the report runs out, as heap's dead_items_alloc's report of the same words does; it reads no catalog ereport(WARNING, (errmsg("disabling parallel option of vacuum on \"%s\" --- cannot vacuum temporary tables in parallel", RelationGetRelationName(rel)))) test/t/062_size_sites.pl: VACUUM warns that a temporary fork table is never vacuumed in parallel vacuum_relstats void Relation rel, BlockNumber pages, double tuples, bool hasindex raises catalog vac_update_relstats updates the relation's pg_class row in place, as heap_vacuum_rel's call does, with every block all-visible and all-frozen (Differences row 10) and no xid horizon of its own vac_update_relstats(rel, pages, tuples, pages, pages, hasindex, InvalidTransactionId, InvalidMultiXactId, NULL, NULL, false) test/t/062_size_sites.pl: VACUUM writes a fork table's pages and rows to pg_class report_vacuum void Relation rel, double live, int64 started raises nocatalog pgstat_report_vacuum counts the vacuum in the relation's pending statistics entry, allocating it the first time, as heap_vacuum_rel's call does; it reads no catalog pgstat_report_vacuum(rel, (PgStat_Counter) live, 0, started) test/t/062_size_sites.pl: VACUUM counts a vacuum of a fork table in the cumulative statistics -fork_named int text *name raises nocatalog forkname_to_number raises pg_relation_size's 22023 for a name no fork has, and text_to_cstring raises only when memory runs out; they read no catalog *out = forkname_to_number(text_to_cstring(name)) test/t/062_size_sites.pl: a fork table's other forks have no size +fork_named int text *name raises nocatalog forkname_to_number raises pg_relation_size's 22023 for a name no fork has, and text_to_cstring raises only when memory runs out; they read no catalog *out = forkname_to_number(text_to_cstring(name)) test/t/062_size_sites.pl: a fork table's free space map fork is its real pages, and its other forks have no size relation_opened void * Oid relid raises catalog RelationIdGetRelation reads the relation's relcache entry, building it from the catalog when it isn't there, and answers NULL for a relation that no longer exists, taking no lock, since the transaction holds one on every relation it wrote *out = RelationIdGetRelation(relid) test/t/062_size_sites.pl: an aborted insert and a rolled-back subtransaction leave a fork table's totals as they were relation_closed void void *rel noraise nocatalog RelationClose only gives the relcache reference back RelationClose((Relation) rel) test/t/062_size_sites.pl: an aborted insert and a rolled-back subtransaction leave a fork table's totals as they were slot_transform DgBytes const TupleTableSlot *slot noraise nocatalog it reads the slot's own buffer slot->tts_ops == &doltgres_slot_ops && ((const DoltgresSlot *) slot)->tuple_len != 0 ? (DgBytes) {((const DoltgresSlot *) slot)->buffer + ((const DoltgresSlot *) slot)->keyed_len + ((const DoltgresSlot *) slot)->key_len + ((const DoltgresSlot *) slot)->tuple_len + ((const DoltgresSlot *) slot)->values_len, ((const DoltgresSlot *) slot)->transform_len} : (DgBytes) {NULL, 0} test/t/067_layouts.pl: a keyed table's DROP COLUMN reads its rows as heap's @@ -510,6 +511,7 @@ slot_missing void TupleTableSlot *slot noraise nocatalog slot_getmissingattrs on rekeyed_rows void Relation rel raises catalog the table's update and its indexes' inserts run the fork's routines and the indexes' access methods, which read catalogs EState *estate = CreateExecutorState(); ResultRelInfo *info = makeNode(ResultRelInfo); Snapshot snapshot = RegisterSnapshot(GetLatestSnapshot()); TupleTableSlot *slot = table_slot_create(rel, NULL); TableScanDesc scan; InitResultRelInfo(info, rel, 1, NULL, 0); ExecOpenIndices(info, false); scan = table_beginscan(rel, snapshot, 0, NULL, SO_NONE); while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) { ItemPointerData tid = slot->tts_tid; TU_UpdateIndexes indexes; MemoryContext caller = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); slot_getallattrs(slot); simple_table_tuple_update(rel, &tid, slot, snapshot, &indexes); if (info->ri_NumIndices > 0 && indexes != TU_None) ExecInsertIndexTuples(info, estate, indexes == TU_Summarizing ? EIIT_ONLY_SUMMARIZING : 0, slot, NIL, NULL); MemoryContextSwitchTo(caller); ResetPerTupleExprContext(estate); } table_endscan(scan); ExecDropSingleTupleTableSlot(slot); ExecCloseIndices(info); UnregisterSnapshot(snapshot); FreeExecutorState(estate) test/t/067_layouts.pl: a keyless table's index scan agrees with its rows after a DROP COLUMN toast_index Oid Oid toast raises catalog toast_get_valid_index opens the TOAST relation and reads its index list from the catalog, under AccessShareLock, which the transaction that made it holds already *out = toast_get_valid_index(toast, AccessShareLock) test/t/057_identity.pl: a CREATE TABLE's TOAST pair reads in its transaction index_info_unique bool struct IndexInfo *info noraise nocatalog it reads a field of the build's own IndexInfo info->ii_Unique test/t/045_btree.pl: a VACUUM FULL rebuilds a unique index a failed CIC left invalid without its unique check, and a DELETE after leaves the index that isn't ready alone, as heap's twin's +index_info_concurrent bool struct IndexInfo *info noraise nocatalog it reads a field of the build's own IndexInfo info->ii_Concurrent test/t/045_btree.pl: a REINDEX CONCURRENTLY computes an indexed row's expression once, as heap's validation does index_recorded bool Relation rel noraise nocatalog it reads the index's own pg_index row in its relcache entry !heap_attisnull(rel->rd_indextuple, Anum_pg_index_indexprs, NULL) || !heap_attisnull(rel->rd_indextuple, Anum_pg_index_indpred, NULL) test/t/045_btree.pl: a DELETE from a table whose expression index a failed CIC left invalid deletes the row and evaluates no expression, as heap's twin's old_entries_unchanged bool int32 at, int32 index, TupleTableSlot *old_row, TupleTableSlot *new_row raises nocatalog pull_varattnos and bms_add_member allocate, which can raise out of memory, and slot_getallattrs deforms the rows; none reads the catalog { IndexInfo *info = doltgres_old_entries[at].infos[index]; TupleDesc desc = old_row->tts_tupleDescriptor; Bitmapset *attrs = NULL; int member = -1; for (int i = 0; i < info->ii_NumIndexAttrs; i++) if (info->ii_IndexAttrNumbers[i] != 0) attrs = bms_add_member(attrs, info->ii_IndexAttrNumbers[i] - FirstLowInvalidHeapAttributeNumber); pull_varattnos((Node *) info->ii_Expressions, 1, &attrs); pull_varattnos((Node *) info->ii_Predicate, 1, &attrs); slot_getallattrs(old_row); slot_getallattrs(new_row); *out = true; while ((member = bms_next_member(attrs, member)) >= 0) { int column = member + FirstLowInvalidHeapAttributeNumber - 1; Form_pg_attribute att; if (column < 0) { *out = false; break; } att = TupleDescAttr(desc, column); if (old_row->tts_isnull[column] != new_row->tts_isnull[column] || (!old_row->tts_isnull[column] && !datumIsEqual(old_row->tts_values[column], new_row->tts_values[column], att->attbyval, att->attlen))) { *out = false; break; } } bms_free(attrs); } test/t/045_btree.pl: an UPDATE of a column a volatile expression index doesn't read calls its function no time, and one of a column it reads calls it once, as heap's twin's register_key_settle void raises nocatalog RegisterXactCallback allocates its entry in TopMemoryContext, which raises only when memory runs out; it reads no catalog RegisterXactCallback(doltgres_key_settle_commit, NULL) test/t/047_rewrite.pl: a drop through performDeletion from a test function has its table's key settled by commit @@ -522,3 +524,7 @@ oldest_nonremovable uint32 Relation rel raises nocatalog GetOldestNonRemovableTr rel_att_storage uint8 Relation rel, int attnum noraise nocatalog it reads the relation's tuple descriptor, which its relcache entry holds (uint8) TupleDescAttr(RelationGetDescr(rel), attnum - 1)->attstorage test/t/038_insert.pl: an INSERT ... SELECT of out-of-line values into a PLAIN column of a table with no TOAST relation stores them inline as heap's twin does transaction_snapshot_push void raises nocatalog GetTransactionSnapshot reads the proc array and PushActiveSnapshot copies it into memory it allocates, neither reading a catalog PushActiveSnapshot(GetTransactionSnapshot()) test/t/062_fold_trigger.pl: while its launcher is held, a standby's slot keeps a comparator-keyed table's first reached point unfolded snapshot_pop void noraise nocatalog PopActiveSnapshot only frees the top of the backend's snapshot stack PopActiveSnapshot() test/t/062_fold_trigger.pl: while its launcher is held, a standby's slot keeps a comparator-keyed table's first reached point unfolded +read_barrier void noraise nocatalog pg_read_barrier only orders the loads around it pg_read_barrier() test/t/071_spill_reads.pl: a scan held across a fold that applied its spilled rows reads every row +atomic_read_u32 uint32 pg_atomic_uint32 *atomic noraise nocatalog pg_atomic_read_u32 only reads pg_atomic_read_u32(atomic) test/t/017_pages.pl: a paused free pass goes on when woken +fork_blocks uint32 Relation rel, int forknum raises nocatalog smgrexists and smgrnblocks open the relation's fork's file through md.c and raise only on an I/O error; they read no catalog *out = smgrexists(RelationGetSmgr(rel), forknum) ? smgrnblocks(RelationGetSmgr(rel), forknum) : 0 test/t/062_size_sites.pl: a fork table's free space map fork is its real pages, and its other forks have no size +aborted_xid uint32 XLogReaderState *record noraise nocatalog ParseAbortRecord only reads the record's bytes { uint8 info = XLogRecGetInfo(record) & XLOG_XACT_OPMASK; result = InvalidTransactionId; if (info == XLOG_XACT_ABORT) result = XLogRecGetXid(record); else if (info == XLOG_XACT_ABORT_PREPARED) { xl_xact_parsed_abort parsed; ParseAbortRecord(XLogRecGetInfo(record), (xl_xact_abort *) XLogRecGetData(record), &parsed); result = parsed.twophase_xid; } } test/t/053_spill.pl: a standby that replayed an aborted transaction's spills reads only the committed rows diff --git a/src/doltgres/pg/entries.list b/src/doltgres/pg/entries.list index 3cf839a98ddd3..d53c106c729a4 100644 --- a/src/doltgres/pg/entries.list +++ b/src/doltgres/pg/entries.list @@ -154,6 +154,7 @@ doltgres_index_blocks bool Relation relation, ForkNumber forkNum, BlockNumber *b doltgres_relation_size_added int64 Relation rel, text *forkName, int64 size error catalog it opens the relation's storage state through the backend's repository, and an index's answer reads its table's pg_class row test/t/062_size_sites.pl: a fork index is whole pages, its metapage and at least one leaf, and every size function sums its relations doltgres_database_size int64 Oid dbOid, int64 size error nocatalog it reads the repository's directories through fd.c's directory functions and stat test/t/062_size_sites.pl: pg_database_size is the database directory's files and its repository's doltgres_tablespace_size int64 Oid tblspcOid, int64 size error nocatalog it reads the tablespace's directory of repositories through fd.c's directory functions and stat test/t/062_size_sites.pl: pg_tablespace_size adds every repository's files to the default tablespace's +doltgres_tablespace_emptied void Oid tablespaceoid, bool redo error nocatalog DropTableSpace calls it before it takes TablespaceCreateLock, so its wait answers cancel, and redo calls it before its first removal; it reads the tablespace's directories through fd.c and asks the collector worker, reading no catalog test/t/041_database.pl: DROP TABLESPACE of a location a fork table left empties the fork's packs out of it doltgres_altering void Relation relation, AlterTableCmd *cmd, int rewrite error catalog ATExecCmd runs each subcommand inside the altering transaction, before the subcommand changes the catalog test/t/067_layouts.pl: a keyed table's DROP COLUMN reads its rows as heap's doltgres_column_dropped void Oid relid, int attnum error catalog doDeletion runs inside the dropping transaction, after the column's catalog row is marked dropped and before the command counter makes that visible test/t/067_layouts.pl: a keyless table's columns DROP TYPE, DROP DOMAIN, and DROP EXTENSION cascade to read as heap's doltgres_toast_created void Relation relation, Oid toast error catalog create_toast_table runs inside the transaction that makes the TOAST relation, after the table's pg_class row names it test/t/057_identity.pl: a CREATE TABLE's TOAST pair reads in its transaction @@ -165,3 +166,9 @@ doltgres_key_settle_table void Relation rel error catalog it compares the table' doltgres_key_settle_commit void XactEvent event, void *arg error catalog before a commit or a prepare it settles every table still named, which reads the catalog, and at any other event it does nothing test/t/047_rewrite.pl: a drop through performDeletion from a test function has its table's key settled by commit doltgres_new_heap void Oid new_table, Relation old_table error catalog it reads the old table's primary key from the catalog test/t/047_rewrite.pl: CLUSTER gives a table's rows the tids heap's rewrite gives them in each of its orders doltgres_backup_checkpointed void uint64 checkpoint error nocatalog do_pg_backup_start calls it after its checkpoint, in its error cleanup's block, where a raise runs do_pg_abort_backup test/t/056_collect.pl: a standby made from a base backup taken while a fold writes a pack starts and reads every row +doltgres_visibility_counted bool Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen error catalog it opens the table's storage state through the backend's repository, which reads the repository's newest manifest, and its rule reads the relation's relcache entry test/t/062_size_sites.pl: CREATE INDEX, REINDEX, and ANALYZE keep a fork table's every block all-visible and all-frozen +doltgres_index_scan_io void Oid indexoid, bool indexonly, double tuples_fetched, double selectivity, double loop_count, double random_page_cost, double seq_page_cost, double cpu_operator_cost, double *max_io, double *min_io, double *rand_heap_pages error catalog the planner costs an index path inside a transaction, where the fork opens the index and its table, whose locks the planner holds, and their storage's state through the backend's repository, and reads the index's keys against the table's test/t/131_scan_costs.pl: an index scan of a fork table costs the fork's descents and leaves, an index-only scan none of the table +doltgres_bitmap_page_cost void Oid relid, double tuples_fetched, double pages_fetched, BlockNumber pages, double maxentries, double random_page_cost, double cpu_operator_cost, double *cost_per_page error catalog the planner costs a bitmap path inside a transaction, where the fork opens the table, whose lock the planner holds, and its storage's state through the backend's repository test/t/131_scan_costs.pl: a bitmap heap scan of a fork table costs a tuple id resolution and a descent a row +doltgres_heap_only_refused bool Relation rel error catalog the check reads the relation's relcache entry and, for an index, its table's and the index's method test/t/132_heap_only_contrib.pl: each heap-only contrib function refuses a fork relation with its own check's error +doltgres_visibility_kept bool Relation rel error catalog the check reads the relation's relcache entry test/t/132_heap_only_contrib.pl: pg_visibility answers every block of a fork table all-visible and all-frozen +doltgres_visibility_bits int Relation rel, BlockNumber blkno, int mapbits error catalog it opens the table's storage state through the backend's repository, which reads the repository's newest manifest, and its rule reads the relation's relcache entry test/t/132_heap_only_contrib.pl: pg_visibility answers every block of a fork table all-visible and all-frozen diff --git a/src/doltgres/pg/test-entries.list b/src/doltgres/pg/test-entries.list index dfed8724a0f51..2b64624167ef7 100644 --- a/src/doltgres/pg/test-entries.list +++ b/src/doltgres/pg/test-entries.list @@ -69,3 +69,7 @@ doltgres_test_epochs Datum error nocatalog it reads the epoch lock's words in s doltgres_test_base_levels int Relation target error catalog it opens the relation's storage's state through the backend's repository and walks its base tree test/t/045_btree.pl: a fork btree index answers its base tree's levels above its leaves, a primary key's index its table's tree's, and 0 for an empty base doltgres_test_totals Datum Relation target error catalog it opens the target's storage's state through the backend's repository, which reads the repository's newest manifest test/t/062_size_sites.pl: an aborted insert and a rolled-back subtransaction leave a fork table's totals as they were doltgres_test_count Datum Relation target error catalog it opens the target's storage's state through the backend's repository and reads its table's key from the catalog for the count's fold test/t/062_size_sites.pl: an aborted insert and a rolled-back subtransaction leave a fork table's totals as they were +doltgres_test_scan_shape Datum Relation target error catalog it opens the target's storage's state through the backend's repository, which reads the repository's newest manifest test/t/131_scan_costs.pl: an index scan of a fork table costs the fork's descents and leaves, an index-only scan none of the table +doltgres_test_base_points Datum Relation target error catalog it opens the target's storage's state through the backend's repository, which reads the repository's newest manifest test/t/047_rewrite.pl: a standby's base of a moved table has its primary's root, fold point, and read point +doltgres_test_identity_versions Datum int32 kind, Oid number error nocatalog it reads the working state's delta and its identity spills under a hold, reading no catalog test/t/133_identity_spill.pl: a restart after a checkpoint that listed a running transaction's identity spill holds each replayed key's version once +doltgres_test_save Datum error nocatalog it saves the backend's database's repository outside a checkpoint, as a states' pool's make does, through its states and identity tree in shared memory, reading no catalog test/t/134_saves.pl: a save's manifest names a fold's base only once the base's record is flushed diff --git a/src/doltgres/pghost/c/delta.c b/src/doltgres/pghost/c/delta.c index d94109445e555..a0a3a45f3cea6 100644 --- a/src/doltgres/pghost/c/delta.c +++ b/src/doltgres/pghost/c/delta.c @@ -216,11 +216,18 @@ pghost_delta_slot_of(uint32 aged, uint32 index) return STD_ADD(u32, at, index, 112, 5); } -/* pghost_delta_stamp_of returns the place among a record's atomics of the epoch of its retirement: its last. */ +/* pghost_delta_stamp_of returns the place among a record's atomics of the epoch of its retirement: the one before its last. */ static size_t pghost_delta_stamp_of(size_t kind) { - return STD_SUB(usize, shmem_records_atomics(kind), 1, 117, 5); + return STD_SUB(usize, shmem_records_atomics(kind), 2, 117, 5); +} + +/* pghost_delta_link_of returns the place among a record's atomics of its link in its kind's retired list: its last (bugs row 309). */ +static size_t +pghost_delta_link_of(size_t kind) +{ + return shmem_records_atomics(kind) - 1; } /* pghost_delta_Pages_Directory_slots is the directory's count of slots. */ @@ -801,6 +808,101 @@ pghost_delta_Pages_Directory_unmade(void *self, uint32 kind, uint32 index) std_RwLockWriteGuard_drop(&made); } +/* pghost_delta_Pages_Directory_retired_head loads the retired list's head. */ +static uint64 +pghost_delta_Pages_Directory_retired_head(void *self) +{ + (void) self; + return shmem_delta_retired(); +} + +/* pghost_delta_Pages_Directory_retired_swap stores the retired list's head over the one loaded. */ +static bool +pghost_delta_Pages_Directory_retired_swap(void *self, uint64 loaded, uint64 word) +{ + (void) self; + return shmem_delta_retired_swap(loaded, word); +} + +/* pghost_delta_Pages_Directory_link loads a slot's link in the retired list. */ +static uint32 +pghost_delta_Pages_Directory_link(void *self, uint32 slot) +{ + (void) self; + return shmem_delta_link(slot); +} + +/* pghost_delta_Pages_Directory_set_link stores a slot's link in the retired list. */ +static void +pghost_delta_Pages_Directory_set_link(void *self, uint32 slot, uint32 value) +{ + (void) self; + shmem_delta_set_link(slot, value); +} + +/* pghost_delta_Pages_Directory_record_retired_head loads an aged kind's retired list's head. */ +static uint64 +pghost_delta_Pages_Directory_record_retired_head(void *self, uint32 kind) +{ + (void) self; + return shmem_records_retired(pghost_delta_kind_of(kind)); +} + +/* pghost_delta_Pages_Directory_record_retired_swap stores an aged kind's retired list's head over the one loaded. */ +static bool +pghost_delta_Pages_Directory_record_retired_swap(void *self, uint32 kind, uint64 loaded, uint64 word) +{ + (void) self; + return shmem_records_retired_swap(pghost_delta_kind_of(kind), loaded, word); +} + +/* pghost_delta_Pages_Directory_record_link loads an aged record's link in its kind's retired list. */ +static uint64 +pghost_delta_Pages_Directory_record_link(void *self, uint32 kind, uint32 index) +{ + size_t region_kind = pghost_delta_kind_of(kind); + ShmemRecordsOp load = {SHMEM_RECORDS_OP_LOAD}; + + (void) self; + return shmem_records_atomic(region_kind, index, pghost_delta_link_of(region_kind), load); +} + +/* pghost_delta_Pages_Directory_set_record_link stores an aged record's link in its kind's retired list by exchanges. */ +static void +pghost_delta_Pages_Directory_set_record_link(void *self, uint32 kind, uint32 index, uint64 value) +{ + size_t region_kind = pghost_delta_kind_of(kind); + ShmemRecordsOp load = {SHMEM_RECORDS_OP_LOAD}; + uint64 held = shmem_records_atomic(region_kind, index, pghost_delta_link_of(region_kind), load); + + (void) self; + for (;;) + { + ShmemRecordsOp exchange = {SHMEM_RECORDS_OP_EXCHANGE, {.exchange = {held, value}}}; + uint64 before = shmem_records_atomic(region_kind, index, pghost_delta_link_of(region_kind), exchange); + + if (before == held) + return; + held = before; + } +} + +/* pghost_delta_Pages_Directory_lock_pass takes the free pass's lock. */ +static void +pghost_delta_Pages_Directory_lock_pass(void *self) +{ + (void) self; + shmem_delta_lock_pass(); +} + +/* pghost_delta_Pages_Directory_unlock_pass releases the free pass's lock. */ +static void +pghost_delta_Pages_Directory_unlock_pass(void *self) +{ + (void) self; + shmem_delta_unlock_pass(); +} + /* PGHOST_DELTA_PAGES_DIRECTORY_OPS is Pages' Directory table. */ static const HostDeltaDirectoryDirectoryOps PGHOST_DELTA_PAGES_DIRECTORY_OPS = { .slots = pghost_delta_Pages_Directory_slots, @@ -858,6 +960,16 @@ static const HostDeltaDirectoryDirectoryOps PGHOST_DELTA_PAGES_DIRECTORY_OPS = { .clear = pghost_delta_Pages_Directory_clear, .made = pghost_delta_Pages_Directory_made, .unmade = pghost_delta_Pages_Directory_unmade, + .retired_head = pghost_delta_Pages_Directory_retired_head, + .retired_swap = pghost_delta_Pages_Directory_retired_swap, + .link = pghost_delta_Pages_Directory_link, + .set_link = pghost_delta_Pages_Directory_set_link, + .record_retired_head = pghost_delta_Pages_Directory_record_retired_head, + .record_retired_swap = pghost_delta_Pages_Directory_record_retired_swap, + .record_link = pghost_delta_Pages_Directory_record_link, + .set_record_link = pghost_delta_Pages_Directory_set_record_link, + .lock_pass = pghost_delta_Pages_Directory_lock_pass, + .unlock_pass = pghost_delta_Pages_Directory_unlock_pass, }; /* PGHOST_DELTA_DIRECTORY is &Pages as host's sequences take it. */ @@ -1015,11 +1127,53 @@ pghost_delta_ended(uint64 epoch) host_delta_directory_released(PGHOST_DELTA_DIRECTORY, epoch); } -/* pghost_delta_open opens a page under an epoch. */ +/* PGHOST_DELTA_OPENS is how many admitted opens a process keeps, a power of two a name's slot indexes. */ +#define PGHOST_DELTA_OPENS 1024 +StaticAssertDecl((PGHOST_DELTA_OPENS & (PGHOST_DELTA_OPENS - 1)) == 0, + "PGHOST_DELTA_OPENS is a power of two"); + +/* PghostDeltaOpen is an open the directory admitted: the page's name, the hold's epoch, and what it reached. */ +typedef struct PghostDeltaOpen +{ + bool some; + HostDeltaPagePage name; + uint64 epoch; + HostDeltaDirectoryOpened opened; +} PghostDeltaOpen; + +/* + * PGHOST_DELTA_OPENED is this process's last admitted open of each of + * PGHOST_DELTA_OPENS sets of names (bugs row 355). An entry answers only the + * same name, generation included, at the same epoch, and only an admission, + * never a refusal: for one name and epoch the directory's answer never turns + * from admitted to refused while a hold at that epoch can ask, since a + * retirement stamps an epoch at least that of every hold it can be seen + * under, and a slot is freed and given its next generation only once every + * hold at an epoch at or below that stamp has dropped, so no hold begun + * after it has the epoch. Its body and size are fixed while its generation + * holds the slot. The body is an offset into this process's mapping, which a + * process never outlives, so nothing clears the table. + */ +static PghostDeltaOpen PGHOST_DELTA_OPENED[PGHOST_DELTA_OPENS]; + +/* pghost_delta_open opens a page under an epoch, answering a repeat of an admitted open from the process's table. */ bool pghost_delta_open(HostDeltaPagePage name, uint64 epoch, HostDeltaDirectoryOpened *result, HostError *error) { - return host_delta_directory_open(PGHOST_DELTA_DIRECTORY, name, epoch, result, error); + PghostDeltaOpen *open = &PGHOST_DELTA_OPENED[host_delta_page_Page_number(name) & (PGHOST_DELTA_OPENS - 1)]; + + if (open->some && open->name.f0 == name.f0 && open->epoch == epoch) + { + *result = open->opened; + return true; + } + if (!host_delta_directory_open(PGHOST_DELTA_DIRECTORY, name, epoch, result, error)) + return false; + *open = (PghostDeltaOpen) + { + true, name, epoch, *result + }; + return true; } /* pghost_delta_bytes_of returns bytes of an opened page, mapped from the region. */ diff --git a/src/doltgres/pghost/c/lib.c b/src/doltgres/pghost/c/lib.c index ef4c264bc9a74..5711cf272f50a 100644 --- a/src/doltgres/pghost/c/lib.c +++ b/src/doltgres/pghost/c/lib.c @@ -1919,17 +1919,17 @@ pghost_PgHost_Host_refuse_recovery(void *self, const char *detail, size_t detail shmem_delta_refuse(detail, detail_len); } -/* pghost_PgHost_Host_delay_checkpoints holds checkpoints off, loading first the pauses its window runs inside its critical section. */ +/* pghost_PgHost_Host_load_pauses loads pauses' callbacks, as INJECTION_POINT_LOAD does, and does nothing in a build without injection points. */ static bool -pghost_PgHost_Host_delay_checkpoints(void *self, const HostMayRaise *raise, const char *const *pauses, - size_t pauses_len, HostError *error) +pghost_PgHost_Host_load_pauses(void *self, const HostMayRaise *raise, const char *const *pauses, size_t pauses_len, + HostError *error) { - PgPgError error_; - (void) self; #ifdef USE_INJECTION_POINTS for (size_t i = 0, i_len = pauses_len; i < i_len; i++) { + PgPgError error_; + if (!pg_calls_injection_point_load(pghost_pg_raise(raise), pauses[i], &error_)) { *error = pghost_held(&error_); @@ -1937,9 +1937,23 @@ pghost_PgHost_Host_delay_checkpoints(void *self, const HostMayRaise *raise, cons } } #else + (void) raise; (void) pauses; (void) pauses_len; + (void) error; #endif + return true; +} + +/* pghost_PgHost_Host_delay_checkpoints holds checkpoints off, loading first the pauses its window runs inside its critical section. */ +static bool +pghost_PgHost_Host_delay_checkpoints(void *self, const HostMayRaise *raise, const char *const *pauses, + size_t pauses_len, HostError *error) +{ + PgPgError error_; + + if (!pghost_PgHost_Host_load_pauses(self, raise, pauses, pauses_len, error)) + return false; if (!pg_calls_delay_checkpoints(pghost_pg_raise(raise), &error_)) { *error = pghost_held(&error_); @@ -2519,6 +2533,9 @@ pghost_PgHost_Host_atomic(void *self, uint64 place, size_t atomic, HostSharedOp records_op.u.exchange.f0 = op.u.exchange.f0; records_op.u.exchange.f1 = op.u.exchange.f1; break; + case HOST_SHARED_OP_LOAD_ACQUIRE: + records_op.tag = SHMEM_RECORDS_OP_LOAD_ACQUIRE; + break; } pghost_shared_placed(place, &kind, &index); return shmem_records_atomic(kind, index, atomic, records_op); @@ -3088,15 +3105,15 @@ pghost_PgHost_Host_collection_end(void *self) shmem_collection_collection_end(); } -/* pghost_PgHost_Host_collection_ask asks the collector to collect a database and waits for its counts. */ +/* pghost_PgHost_Host_collection_ask asks the collector to collect a database, emptying a tablespace, and waits for its counts. */ static bool -pghost_PgHost_Host_collection_ask(void *self, const HostMayRaise *raise, uint32 database, uint64 *result, - HostError *error) +pghost_PgHost_Host_collection_ask(void *self, const HostMayRaise *raise, uint32 database, uint32 emptied, + uint64 *result, HostError *error) { PgPgError error_; (void) self; - if (!shmem_collection_ask(pghost_pg_raise(raise), database, result, &error_)) + if (!shmem_collection_ask(pghost_pg_raise(raise), database, emptied, result, &error_)) { *error = pghost_held(&error_); return false; @@ -3104,14 +3121,14 @@ pghost_PgHost_Host_collection_ask(void *self, const HostMayRaise *raise, uint32 return true; } -/* pghost_PgHost_Host_collection_asked returns the unanswered ask's word and the database it asks for. */ +/* pghost_PgHost_Host_collection_asked returns the unanswered ask's word, the database it asks for, and the tablespace it empties. */ static bool -pghost_PgHost_Host_collection_asked(void *self, uint64 *result_0, uint32 *result_1) +pghost_PgHost_Host_collection_asked(void *self, uint64 *result_0, uint32 *result_1, uint32 *result_2) { std_TupleU64U32_t asked; (void) self; - if (!shmem_collection_asked_for(&asked)) + if (!shmem_collection_asked_for(&asked, result_2)) return false; *result_0 = asked.f0; *result_1 = asked.f1; @@ -3134,6 +3151,14 @@ pghost_PgHost_Host_collector_started(void *self) shmem_collection_worker_started(); } +/* pghost_PgHost_Host_collector_running reports whether a collector worker has started. */ +static bool +pghost_PgHost_Host_collector_running(void *self) +{ + (void) self; + return shmem_collection_worker_running(); +} + /* pghost_PgHost_Host_database_locked takes a database's lock without waiting, reporting whether it did. */ static bool pghost_PgHost_Host_database_locked(void *self, const HostMayRaise *raise, uint32 database, bool *result, @@ -4138,6 +4163,7 @@ static const HostHostOps PGHOST_PG_HOST_HOST_OPS = { .log = pghost_PgHost_Host_log, .refuse_recovery = pghost_PgHost_Host_refuse_recovery, .delay_checkpoints = pghost_PgHost_Host_delay_checkpoints, + .load_pauses = pghost_PgHost_Host_load_pauses, .resume_checkpoints = pghost_PgHost_Host_resume_checkpoints, .xid_status = pghost_PgHost_Host_xid_status, .xid_in_progress = pghost_PgHost_Host_xid_in_progress, @@ -4244,6 +4270,7 @@ static const HostHostOps PGHOST_PG_HOST_HOST_OPS = { .collection_asked = pghost_PgHost_Host_collection_asked, .collection_answer = pghost_PgHost_Host_collection_answer, .collector_started = pghost_PgHost_Host_collector_started, + .collector_running = pghost_PgHost_Host_collector_running, .database_locked = pghost_PgHost_Host_database_locked, .database_unlocked = pghost_PgHost_Host_database_unlocked, .database_lock_waited = pghost_PgHost_Host_database_lock_waited, diff --git a/src/doltgres/repo/c/lib.c b/src/doltgres/repo/c/lib.c index 060d4647ad381..05c6d830e8b6c 100644 --- a/src/doltgres/repo/c/lib.c +++ b/src/doltgres/repo/c/lib.c @@ -150,14 +150,20 @@ repo_runs_pad_0(void *frame_) } } -/* repo_runs_file reads one file of a manifest's runs from its item, refusing what runs refuses. */ +/* + * repo_runs_file reads one file of a manifest's runs from its item, refusing + * what runs refuses: a spill's xid without its number, its number without its + * xid, or its passing point or residual's point without either. + */ static bool repo_runs_file(const ChunksRecordRecord *item, RepoRunFile *file, ChunksErrorError *error) { - static const uint32 numbers[] = {1, 2, 3, 4, 5, 6, 7}; + static const uint32 numbers[] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; uint64 kind; bool has_xid; bool has_number; + bool has_passed; + bool has_from; if (!repo_known(item, numbers, lengthof(numbers))) return repo_corrupt(error); @@ -168,12 +174,22 @@ repo_runs_file(const ChunksRecordRecord *item, RepoRunFile *file, ChunksErrorErr file->kind = (uint8) kind; has_xid = chunks_record_Record_field(item, 6) != NULL; has_number = chunks_record_Record_field(item, 7) != NULL; - if (has_xid != has_number) + has_passed = chunks_record_Record_field(item, 8) != NULL; + has_from = chunks_record_Record_field(item, 9) != NULL; + if (has_xid != has_number || (!has_xid && (has_passed || has_from))) return repo_corrupt(error); file->has_spilled = has_xid; if (has_xid && (!chunks_record_Record_uint(item, 6, &file->spilled.xid, error) || !repo_narrow(item, 7, &file->spilled.number, error))) return false; + if (has_xid) + { + file->spilled.has_passed = has_passed; + file->spilled.has_from = has_from; + if ((has_passed && !chunks_record_Record_uint(item, 8, &file->spilled.passed, error)) || + (has_from && !chunks_record_Record_uint(item, 9, &file->spilled.from, error))) + return false; + } return repo_narrow(item, 2, &file->tablespace, error) && repo_narrow(item, 3, &file->relation, error) && chunks_record_Record_uint(item, 4, &file->len, error) && chunks_record_Record_uint(item, 5, &file->hash, error); } @@ -346,6 +362,10 @@ repo_Manifest_encode_body_0(void *frame_) { chunks_record_RecordWriter_uint(&frame->item, 6, file->spilled.xid); chunks_record_RecordWriter_uint(&frame->item, 7, file->spilled.number); + if (file->spilled.has_passed) + chunks_record_RecordWriter_uint(&frame->item, 8, file->spilled.passed); + if (file->spilled.has_from) + chunks_record_RecordWriter_uint(&frame->item, 9, file->spilled.from); } std_vec_push_Checked(&frame->files, &frame->item, sizeof(frame->item), _Alignof(ChunksRecordRecordWriter)); frame->item_live = false; @@ -612,10 +632,14 @@ bool repo_Spilled_Debug_fmt(const void *value, StdFormatter *f) { const RepoSpilled *self = value; + StdOptionRef passed = {self->has_passed, {&self->passed, std_u64_Debug_fmt}}; + StdOptionRef from = {self->has_from, {&self->from, std_u64_Debug_fmt}}; StdDebugStruct s = std_fmt_debug_struct(f, "Spilled", strlen("Spilled")); std_DebugStruct_field(&s, "xid", strlen("xid"), &self->xid, std_u64_Debug_fmt); std_DebugStruct_field(&s, "number", strlen("number"), &self->number, std_u32_Debug_fmt); + std_DebugStruct_field(&s, "passed", strlen("passed"), &passed, std_option_Debug_fmt); + std_DebugStruct_field(&s, "from", strlen("from"), &from, std_option_Debug_fmt); return std_DebugStruct_finish(&s); } @@ -623,7 +647,9 @@ repo_Spilled_Debug_fmt(const void *value, StdFormatter *f) bool repo_Spilled_PartialEq_eq(const RepoSpilled *self, const RepoSpilled *other) { - return self->xid == other->xid && self->number == other->number; + return self->xid == other->xid && self->number == other->number && self->has_passed == other->has_passed && + (!self->has_passed || self->passed == other->passed) && self->has_from == other->has_from && + (!self->has_from || self->from == other->from); } /* repo_RunFile_Debug_fmt is RunFile's derived Debug. */ @@ -1845,9 +1871,10 @@ repo_decoded(TreesPublishedRepoId repo, bool has, const chunks_TupleAddressLocat /* * repo_Repo_save_run_0 is save's run under the root slots: it holds and - * publishes the roots the pass wrote, writes the manifest that names them - * if no kept file of the repository was written since the pass's read, and - * takes the runs back out of it. + * publishes the roots the pass wrote, lists every pack whose add the + * repository defers, writes the manifest that names them if no kept file of + * the repository was written since the pass's read, and takes the runs back + * out of it. */ static bool repo_Repo_save_run_inner(repo_Repo_save_frame_0_t *frame, HostIoRegistering *registering, @@ -1904,6 +1931,8 @@ repo_Repo_save_run_inner(repo_Repo_save_frame_0_t *frame, HostIoRegistering *reg } manifest->has_runs = false; frame->written_manifest_live = true; + if (!chunks_packs_backend_Packs_flush(&frame->self->packs, io, frame->reading, error)) + return false; if (!host_last_redo(doors, &manifest->redo, &host_error)) { *error = chunks_error_Error_FromError_from(&host_error); @@ -2094,7 +2123,14 @@ repo_Repo_save_body_0(void *frame_) } if (frame->landed) { + HostError host_error; + repo_Repo_save_pad_0(frame); + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-repo-save-landed", &host_error)) + { + *frame->error = chunks_error_Error_FromError_from(&host_error); + return; + } frame->ok = frame->saved(frame->saved_captures, frame->io, frame->error); return; } @@ -2966,7 +3002,7 @@ repo_tests_runs(HostLsn redo, RepoRuns *result) {.kind = 0, .tablespace = HOST_PG_DEFAULT, .relation = 16384, .len = 4096, .hash = 0xfeed}, {.kind = 3, .tablespace = 0, .relation = 0, .len = 128, .hash = PG_UINT64_MAX}, {.kind = 5, .tablespace = HOST_PG_DEFAULT, .relation = 16384, .len = 512, .hash = 7, .has_spilled = true, - .spilled = {((uint64) 1 << 40) | 742, 3}}, + .spilled = {((uint64) 1 << 40) | 742, 3, true, ((uint64) 1 << 33) | 5, true, ((uint64) 1 << 33) | 9}}, }; result->redo = redo; @@ -3045,7 +3081,7 @@ repo_tests_runs_body_0(void *frame_) { uint32 number; uint64 value; - } listings[] = {{1, 256}, {2, (uint64) 1 << 32}, {3, (uint64) 1 << 32}, {6, 1}, {7, 1}, {8, 1}}; + } listings[] = {{1, 256}, {2, (uint64) 1 << 32}, {3, (uint64) 1 << 32}, {6, 1}, {7, 1}, {8, 1}, {9, 1}}; for (size_t k = 0, k_len = 3; k < k_len; k++) { diff --git a/src/doltgres/shmem/c/collection.c b/src/doltgres/shmem/c/collection.c index 924ceff878020..c702864f0383b 100644 --- a/src/doltgres/shmem/c/collection.c +++ b/src/doltgres/shmem/c/collection.c @@ -49,6 +49,9 @@ static const char SHMEM_COLLECTION_ASKING[] = "versioned collection ask"; /* SHMEM_COLLECTION_DONE is the bit of the ask's word the worker sets once it has written its answer. */ #define SHMEM_COLLECTION_DONE ((uint64) 1 << 32) +/* SHMEM_COLLECTION_HELD is the bit of the ask's word its asker holds while it writes the tablespace the ask empties. */ +#define SHMEM_COLLECTION_HELD ((uint64) 1 << 63) + /* * SHMEM_COLLECTION_PG_WAIT_LWLOCK is Postgres' wait event class of a * lightweight lock's tranche, which the backup's sleep names with the @@ -61,8 +64,9 @@ static const char SHMEM_COLLECTION_ASKING[] = "versioned collection ask"; * backups, the collection's word (one while a collection runs), the condition * variable a backup's start sleeps on, and the tranche its sleep names; then * the ask's word (0, or the asker's process number plus one above DONE, DONE - * once answered, and the database asked for below it), the collector - * worker's process number plus one (0 before it starts), the answer's + * once answered, and the database asked for below it), the tablespace the + * ask empties, the collector worker's process number plus one (0 before it + * starts), the answer's * counts, the condition variable askers sleep on, and the tranche their * sleep names. */ @@ -73,6 +77,7 @@ typedef struct ShmemCollectionMeeting ConditionVariable wake; uint32 tranche; pg_atomic_uint64 asked; + pg_atomic_uint64 emptied; pg_atomic_uint64 worker; pg_atomic_uint64 answer[3]; ConditionVariable answered; @@ -117,6 +122,8 @@ shmem_collection_initialize(const PgMayRaise *raise, PgPgError *error) region = shmem_collection_region(); pg_calls_atomic_init_u64(®ion->asked, 0); region = shmem_collection_region(); + pg_calls_atomic_init_u64(®ion->emptied, 0); + region = shmem_collection_region(); pg_calls_atomic_init_u64(®ion->worker, 0); for (size_t at = 0, at_end = 3; at < at_end; at = std_forward_unchecked_usize_Checked(at, 1)) { @@ -278,6 +285,13 @@ shmem_collection_worker_started(void) pg_calls_atomic_write_u64(&shmem_collection_region()->worker, number); } +/* shmem_collection_worker_running reports whether a collector worker has started, which an ask needs to be answered. */ +bool +shmem_collection_worker_running(void) +{ + return shmem_collection_read(&shmem_collection_region()->worker) != 0; +} + /* shmem_collection_ask_closure_0 is map's closure: an answer's count at a place. */ static uint64 shmem_collection_ask_closure_0(void *captures, size_t at) @@ -339,14 +353,15 @@ shmem_collection_ask_pad_0(void *frame_) /* * shmem_collection_ask asks the collector worker for a full collection of a - * database's repository and sleeps until it answers with the packs it + * database's repository, emptying a tablespace unless that is + * CHUNKS_PACKS_DEFAULT_SPACE, and sleeps until it answers with the packs it * retired, those it copied out, and its live bytes, each * SHMEM_COLLECTION_FAILED when it failed; one ask is out at a time, and a * later asker sleeps until it may write its own. A cancel ends the sleep, * taking back an ask not yet answered, and the collection goes on. */ bool -shmem_collection_ask(const PgMayRaise *raise, uint32 database, uint64 result[3], PgPgError *error) +shmem_collection_ask(const PgMayRaise *raise, uint32 database, uint32 emptied, uint64 result[3], PgPgError *error) { uint64 tmp_1 = (uint64) (uint32) pg_calls_my_proc_number(); uint64 tmp_2 = STD_ADD(u64, tmp_1, 1, 218, 18); @@ -358,11 +373,13 @@ shmem_collection_ask(const PgMayRaise *raise, uint32 database, uint64 result[3], STD_CHECK_ALIGNED(region, _Alignof(ShmemCollectionMeeting), 222, 42); STD_CHECK_NULL_DEREF(region, 222, 42); wait = SHMEM_COLLECTION_PG_WAIT_LWLOCK | region->asking; - while (!shmem_collection_swapped(shmem_collection_asked(), 0, mine)) + while (!shmem_collection_swapped(shmem_collection_asked(), 0, mine | SHMEM_COLLECTION_HELD)) { if (!shmem_collection_slept(raise, wait, false, 0, error)) return false; } + pg_calls_atomic_write_u64(&shmem_collection_region()->emptied, emptied); + (void) shmem_collection_swapped(shmem_collection_asked(), mine | SHMEM_COLLECTION_HELD, mine); worker = shmem_collection_read(&shmem_collection_region()->worker); if (worker != 0) { @@ -409,9 +426,9 @@ shmem_collection_slept(const PgMayRaise *raise, uint32 wait, bool has_mine, uint return false; } -/* shmem_collection_asked_for returns the unanswered ask's word and the database it asks for, the collector worker's read. */ +/* shmem_collection_asked_for returns the unanswered ask's word, the database it asks for, and the tablespace it empties, the collector worker's read. */ bool -shmem_collection_asked_for(std_TupleU64U32_t *result) +shmem_collection_asked_for(std_TupleU64U32_t *result, uint32 *emptied) { uint64 word = shmem_collection_read(shmem_collection_asked()); uint32 database; @@ -421,10 +438,12 @@ shmem_collection_asked_for(std_TupleU64U32_t *result) std_result_unwrap_failed("the word's low half fits a database id", sizeof("the word's low half fits a database id") - 1, &tried, std_TryFromIntError_Debug_fmt, RUST_FILE, 273, 62); - if (word != 0 && (word & SHMEM_COLLECTION_DONE) == 0) + if (word != 0 && (word & (SHMEM_COLLECTION_DONE | SHMEM_COLLECTION_HELD)) == 0) { + pg_calls_read_barrier(); result->f0 = word; result->f1 = database; + *emptied = (uint32) shmem_collection_read(&shmem_collection_region()->emptied); return true; } return false; diff --git a/src/doltgres/shmem/c/delta.c b/src/doltgres/shmem/c/delta.c index bb3bd09550210..af9c2b815b9cb 100644 --- a/src/doltgres/shmem/c/delta.c +++ b/src/doltgres/shmem/c/delta.c @@ -17,9 +17,10 @@ /* * The delta's memory in the server (design item 14's S6): the page * directory, a fixed region of a header, one slot of eight words for each 1 - * KiB of delta_memory, three words for each block of the region, and a word of + * KiB of delta_memory, three words for each block of the region, a word of * the free hint for each 64 slots below the block past the budget's (bugs row - * 150); and the region of blocks, a region of its own that holds the pages' + * 150), and a slot's link in the retired list for each slot (bugs row 309); + * and the region of blocks, a region of its own that holds the pages' * bodies and nothing else (bugs row 116). The postmaster makes both. Postgres * keeps the regions' addresses in pg's C globals, so this module keeps only * numbers and translates them inside a call. @@ -43,16 +44,16 @@ static const char SHMEM_DELTA_WAIT[] = "versioned delta budget"; /* SHMEM_DELTA_REFUSAL is the most bytes of a refusal's detail, its ending zero byte included. */ #define SHMEM_DELTA_REFUSAL ((size_t) 384) -/* SHMEM_DELTA_SLOTS is the offset of the slots in the directory: after the header's ten cache lines. */ -#define SHMEM_DELTA_SLOTS ((size_t) 640) +/* SHMEM_DELTA_SLOTS is the offset of the slots in the directory: after the header's eleven cache lines. */ +#define SHMEM_DELTA_SLOTS ((size_t) 704) /* * ShmemDeltaHeader is the directory's fixed part, on its first cache lines: * the count of frees, the hand of an allocation's sweep, the allocation lock, * the condition variable of the budget's wait with its wait event, the count * of slots, the overshoot word, the region's words, lock, and count of - * blocks, the fold launcher's asks and process, and a standby's root - * mismatch. + * blocks, the fold launcher's asks and process, a standby's root mismatch, + * and the retired list's head and the free pass's lock. */ typedef struct ShmemDeltaHeader { @@ -111,6 +112,12 @@ typedef struct ShmemDeltaHeader */ pg_atomic_uint64 refused; + /* + * retired is the first slot plus one of the retired list, the slots the + * free pass has yet to free, 0 for none (bugs row 309). + */ + pg_atomic_uint64 retired; + /* * refusal is the detail of the startup process's FATAL, ended by a zero * byte, once refused is 2. @@ -119,6 +126,8 @@ typedef struct ShmemDeltaHeader LWLock lock; /* region_lock is the region's lock, held only for a block's take or give. */ LWLock region_lock; + /* pass_lock is the free pass's lock, which runs one pass at a time. */ + LWLock pass_lock; ConditionVariable wait; uint32 wait_event; uint32 slots; @@ -214,6 +223,22 @@ shmem_delta_hint_word(uint32 word) return (pg_atomic_uint64 *) STD_PTR_AT(pg_calls_delta_directory(), STD_ADD(usize, blocks, tmp_5, 134, 53), 1); } +/* + * shmem_delta_link_word returns the address of a slot's link in the retired + * list, which follow the free hint's words: the next slot plus one, 0 for the + * list's last (bugs row 309). + */ +static pg_atomic_uint32 * +shmem_delta_link_word(uint32 slot) +{ + size_t slots = (size_t) shmem_delta_slots(); + size_t words = (size_t) shmem_delta_blocks() * SHMEM_DELTA_BLOCK_WORDS + shmem_delta_hints(slots); + size_t links = SHMEM_DELTA_SLOTS + slots * sizeof(ShmemDeltaSlot) + words * sizeof(pg_atomic_uint64); + + Assert(slot < slots); + return (pg_atomic_uint32 *) STD_PTR_AT(pg_calls_delta_directory(), links + (size_t) slot * sizeof(pg_atomic_uint32), 1); +} + /* * shmem_delta_hints returns the count of the free hint's words of a directory * of a count of slots: one for each SHMEM_DELTA_HINTED of those below the @@ -230,8 +255,8 @@ shmem_delta_hints(size_t slots) /* * shmem_delta_request asks Postgres for the delta's two regions, in the * postmaster's request callback: the region of blocks, of its bytes, and the - * directory, of its header, its slots, its blocks' words, and the free - * hint's words. + * directory, of its header, its slots, its blocks' words, the free hint's + * words, and the retired list's links. */ bool shmem_delta_request(const PgMayRaise *raise, size_t slots, size_t blocks, size_t bytes, PgPgError *error) @@ -242,16 +267,17 @@ shmem_delta_request(const PgMayRaise *raise, size_t slots, size_t blocks, size_t size_t tmp_3 = STD_MUL(usize, slots, sizeof(ShmemDeltaSlot), 147, 45); size_t tmp_4 = STD_ADD(usize, SHMEM_DELTA_SLOTS, tmp_3, 147, 37); size_t tmp_5 = STD_ADD(usize, tmp_4, words, 147, 37); + size_t links = std_mul_usize(slots, sizeof(pg_atomic_uint32), __FILE__, __LINE__, 0); - return pg_calls_request_delta(raise, bytes, tmp_5, error); + return pg_calls_request_delta(raise, bytes, std_add_usize(tmp_5, links, __FILE__, __LINE__, 0), error); } /* * shmem_delta_initialize sets the delta's directory up in the postmaster's * initialization callback, after the fork's region: the header, every slot - * free at its first generation, every block free and never taken, and every - * bit of the free hint set, since every slot is free. The region's bodies are - * zeroed as each is taken. + * free at its first generation, every block free and never taken, every bit + * of the free hint set, since every slot is free, and the retired list empty. + * The region's bodies are zeroed as each is taken. */ bool shmem_delta_initialize(const PgMayRaise *raise, size_t slots, size_t blocks, PgPgError *error) @@ -284,8 +310,10 @@ shmem_delta_initialize(const PgMayRaise *raise, size_t slots, size_t blocks, PgP pg_calls_atomic_init_u64(&header->asked, 0); pg_calls_atomic_init_u64(&header->launcher, 0); pg_calls_atomic_init_u64(&header->refused, 0); + pg_calls_atomic_init_u64(&header->retired, 0); pg_calls_lwlock_initialize(&header->lock, tranche); pg_calls_lwlock_initialize(&header->region_lock, tranche); + pg_calls_lwlock_initialize(&header->pass_lock, tranche); pg_calls_cv_init(&header->wait); header->wait_event = wait_event; header->slots = (uint32) slots; @@ -299,6 +327,8 @@ shmem_delta_initialize(const PgMayRaise *raise, size_t slots, size_t blocks, PgP tmp_5 = STD_ADD(usize, tmp_4, words, 185, 43); std_ptr_write_bytes(STD_PTR_AT(shmem_delta_slot(0), tmp_5, 1), 0xff, STD_MUL(usize, shmem_delta_hints(slots), 8, 185, 96), 1, 1); + for (uint32 slot = 0; slot < (uint32) slots; slot++) + pg_calls_atomic_init_u32(shmem_delta_link_word(slot), 0); return true; } @@ -543,6 +573,51 @@ shmem_delta_hand(void) return (uint32) STD_REM(u64, hand, tmp_1, 417, 6); } +/* shmem_delta_retired loads the retired list's head: its first slot plus one, 0 for none. */ +uint64 +shmem_delta_retired(void) +{ + return pg_calls_atomic_read_u64(&shmem_delta_header()->retired); +} + +/* shmem_delta_retired_swap stores the retired list's head over the one loaded, reporting whether it was still there. */ +bool +shmem_delta_retired_swap(uint64 loaded, uint64 word) +{ + uint64 seen = loaded; + + pg_calls_atomic_exchange_u64(&shmem_delta_header()->retired, &seen, word); + return seen == loaded; +} + +/* shmem_delta_link loads a slot's link in the retired list: the next slot plus one, 0 for the last. */ +uint32 +shmem_delta_link(uint32 slot) +{ + return pg_calls_atomic_read_u32(shmem_delta_link_word(slot)); +} + +/* shmem_delta_set_link stores a slot's link in the retired list. */ +void +shmem_delta_set_link(uint32 slot, uint32 value) +{ + pg_calls_atomic_write_u32(shmem_delta_link_word(slot), value); +} + +/* shmem_delta_lock_pass takes the free pass's lock exclusively, which can't fail. */ +void +shmem_delta_lock_pass(void) +{ + pg_calls_lwlock_acquire_sure(&shmem_delta_header()->pass_lock, LW_EXCLUSIVE); +} + +/* shmem_delta_unlock_pass releases the free pass's lock, which its caller holds. */ +void +shmem_delta_unlock_pass(void) +{ + pg_calls_lwlock_release(&shmem_delta_header()->pass_lock); +} + /* shmem_delta_lock takes the allocation lock exclusively. */ bool shmem_delta_lock(const PgMayRaise *raise, PgPgError *error) @@ -771,6 +846,9 @@ shmem_delta_body_word(uint64 pointer, uint64 index, ShmemRecordsOp op) pg_calls_atomic_exchange_u64(word, &seen, op.u.exchange.f1); return seen; } + case SHMEM_RECORDS_OP_LOAD_ACQUIRE: + pg_calls_read_barrier(); + return pg_calls_atomic_read_u64(word); } pg_unreachable(); } diff --git a/src/doltgres/shmem/c/records.c b/src/doltgres/shmem/c/records.c index 887fb9005ff5d..64569f43546f7 100644 --- a/src/doltgres/shmem/c/records.c +++ b/src/doltgres/shmem/c/records.c @@ -78,11 +78,16 @@ typedef struct ShmemRecordsSpot ShmemRecordsForm form; } ShmemRecordsSpot; -/* ShmemRecordsPlaced is a kind in the region: its spot, and the hand its makes sweep from. */ +/* + * ShmemRecordsPlaced is a kind in the region: its spot, the hand its makes + * sweep from, and for an aged kind the head of its retired list, its first + * record plus one, 0 for none (bugs row 309). + */ typedef struct ShmemRecordsPlaced { ShmemRecordsSpot spot; pg_atomic_uint64 hand; + pg_atomic_uint64 retired; } ShmemRecordsPlaced; /* @@ -373,18 +378,26 @@ shmem_records_part(size_t offset, size_t index, size_t size) return STD_PTR_AT(base, std_wrapping_add_usize(0, tmp_2), 1); } +/* ShmemRecordsResolved is a kind as a process resolves it: its place and counts, and its records' layout. */ +typedef struct ShmemRecordsResolved +{ + ShmemRecordsSpot spot; + ShmemRecordsLayout layout; +} ShmemRecordsResolved; + /* ShmemRecordsMapped is a record in this process's mapping of the region, with its kind's counts and its layout. */ typedef struct ShmemRecordsMapped { uint8 *base; - ShmemRecordsSpot spot; - ShmemRecordsLayout layout; + const ShmemRecordsSpot *spot; + const ShmemRecordsLayout *layout; } ShmemRecordsMapped; /* shmem_records_Mapped_of returns the record at an index among a kind's records, refusing an index past the kind's count. */ static ShmemRecordsMapped -shmem_records_Mapped_of(const ShmemRecordsSpot *spot, size_t index) +shmem_records_Mapped_of(const ShmemRecordsResolved *resolved, size_t index) { + const ShmemRecordsSpot *spot = &resolved->spot; ShmemRecordsMapped result; size_t tmp_1; @@ -392,8 +405,8 @@ shmem_records_Mapped_of(const ShmemRecordsSpot *spot, size_t index) STD_PANIC(419, 9, "a shared record's number is past its kind's count"); tmp_1 = STD_MUL(usize, index, (size_t) spot->size, 420, 53); result.base = (uint8 *) shmem_records_part((size_t) spot->at, tmp_1, 1); - result.spot = *spot; - result.layout = shmem_records_layout(spot); + result.spot = spot; + result.layout = &resolved->layout; return result; } @@ -419,41 +432,41 @@ shmem_records_Mapped_part(const ShmemRecordsMapped *self, size_t at, uint32 coun static LWLock * shmem_records_Mapped_lock(const ShmemRecordsMapped *self, size_t index) { - return shmem_records_Mapped_part(self, 0, self->spot.locks, index, sizeof(LWLock)); + return shmem_records_Mapped_part(self, 0, self->spot->locks, index, sizeof(LWLock)); } /* shmem_records_Mapped_flag returns a flag's address. */ static ShmemRecordsFlagSlot * shmem_records_Mapped_flag(const ShmemRecordsMapped *self, size_t index) { - return shmem_records_Mapped_part(self, self->layout.flags, self->spot.flags, index, sizeof(ShmemRecordsFlagSlot)); + return shmem_records_Mapped_part(self, self->layout->flags, self->spot->flags, index, sizeof(ShmemRecordsFlagSlot)); } /* shmem_records_Mapped_atomic returns an atomic's address. */ static pg_atomic_uint64 * shmem_records_Mapped_atomic(const ShmemRecordsMapped *self, size_t index) { - return shmem_records_Mapped_part(self, self->layout.atomics, self->spot.atomics, index, sizeof(pg_atomic_uint64)); + return shmem_records_Mapped_part(self, self->layout->atomics, self->spot->atomics, index, sizeof(pg_atomic_uint64)); } /* shmem_records_Mapped_blob returns the address of a blob's length, which its bytes follow. */ static uint64 * shmem_records_Mapped_blob(const ShmemRecordsMapped *self, size_t index) { - if (!(index < (size_t) self->spot.blobs)) + if (!(index < (size_t) self->spot->blobs)) STD_PANIC(447, 9, "a shared record's part index is past its shape's count"); std_index_check(index, SHMEM_RECORDS_BLOBS, RUST_FILE, 448, 32); - return STD_PTR_AT(self->base, self->layout.blobs[index], 1); + return STD_PTR_AT(self->base, self->layout->blobs[index], 1); } /* shmem_records_Mapped_array returns the address of an array's count, which its name and then its entries follow. */ static pg_atomic_uint64 * shmem_records_Mapped_array(const ShmemRecordsMapped *self, size_t index) { - if (!(index < (size_t) self->spot.arrays)) + if (!(index < (size_t) self->spot->arrays)) STD_PANIC(453, 9, "a shared record's part index is past its shape's count"); std_index_check(index, SHMEM_RECORDS_ARRAYS, RUST_FILE, 454, 32); - return STD_PTR_AT(self->base, self->layout.arrays[index], 1); + return STD_PTR_AT(self->base, self->layout->arrays[index], 1); } /* shmem_records_Mapped_entries returns the address of an array's first entry. */ @@ -470,12 +483,12 @@ shmem_records_Mapped_entry(const ShmemRecordsMapped *self, size_t array, uint64 uint8 *first = shmem_records_Mapped_entries(self, array); std_index_check(array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 465, 28); - if (entry < (uint64) self->spot.entries[array]) + if (entry < (uint64) self->spot->entries[array]) { size_t tmp_1; std_index_check(array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 466, 58); - tmp_1 = STD_MUL(usize, (size_t) entry, (size_t) self->spot.widths[array], 466, 41); + tmp_1 = STD_MUL(usize, (size_t) entry, (size_t) self->spot->widths[array], 466, 41); return STD_PTR_AT(first, tmp_1, 1); } return NULL; @@ -484,6 +497,58 @@ shmem_records_Mapped_entry(const ShmemRecordsMapped *self, size_t array, uint64 /* shmem_records_record returns the record at an index among a kind's records. */ static ShmemRecordsMapped shmem_records_record(size_t kind, uint32 index); +/* + * SHMEM_RECORDS_RESOLVED holds, once this process first reaches a record, each + * kind's place, counts, and layout as offsets from the process's own mapping + * of the region, a Vec read-only after it is made: the postmaster + * fixes the places for its life, a process attaches once, and the region's + * initialize, which a crash restart runs again, clears it (bugs row 355). + */ +static StdVec SHMEM_RECORDS_RESOLVED = STD_VEC_NEW; + +/* shmem_records_unresolved clears this process's resolutions, for the region's initialize. */ +static void +shmem_records_unresolved(void) +{ + std_vec_free_Checked(&SHMEM_RECORDS_RESOLVED, sizeof(ShmemRecordsResolved), _Alignof(ShmemRecordsResolved)); + std_vec_new(&SHMEM_RECORDS_RESOLVED, sizeof(ShmemRecordsResolved), _Alignof(ShmemRecordsResolved)); +} + +/* + * shmem_records_resolved returns a kind as this process resolved it, the + * first call resolving every kind of the region from its places, refusing a + * kind the region doesn't hold, which only a caller that passes a kind this + * module didn't give it could ask for. + */ +static const ShmemRecordsResolved * +shmem_records_resolved(size_t kind) +{ + if (SHMEM_RECORDS_RESOLVED.len == 0) + { + ShmemControl *control = shmem_records_control(); + size_t kinds; + + STD_CHECK_ALIGNED(control, _Alignof(ShmemControl), 321, 24); + STD_CHECK_NULL_DEREF(control, 321, 24); + kinds = (size_t) control->kinds; + std_vec_with_capacity_Checked(&SHMEM_RECORDS_RESOLVED, kinds, sizeof(ShmemRecordsResolved), + _Alignof(ShmemRecordsResolved)); + for (size_t each = 0; each < kinds; each++) + { + ShmemRecordsResolved resolved; + + resolved.spot = ((ShmemRecordsPlaced *) shmem_records_part(SHMEM_RECORDS_PLACED, each, + sizeof(ShmemRecordsPlaced)))->spot; + resolved.layout = shmem_records_layout(&resolved.spot); + std_vec_push_Checked(&SHMEM_RECORDS_RESOLVED, &resolved, sizeof(resolved), + _Alignof(ShmemRecordsResolved)); + } + } + if (!(kind < SHMEM_RECORDS_RESOLVED.len)) + STD_PANIC(321, 9, "a shared record's kind is past the region's kinds"); + return &STD_VEC_AT(&SHMEM_RECORDS_RESOLVED, ShmemRecordsResolved, kind); +} + /* shmem_records_initialize_frame_0_t is initialize's state across its pad: its arguments and the plan. */ typedef struct shmem_records_initialize_frame_0_t { @@ -507,6 +572,7 @@ shmem_records_initialize_body_0(void *frame_) size_t procs = (size_t) pg_calls_max_procs(); uint32 flag_wait; + shmem_records_unresolved(); frame->plan = shmem_records_plan(frame->kinds, frame->kinds_len, procs); frame->plan_live = true; if (!pg_calls_wait_event_new(frame->raise, SHMEM_RECORDS_FLAG_WAIT, &flag_wait, frame->error)) @@ -535,16 +601,20 @@ shmem_records_initialize_body_0(void *frame_) size_t kind = count; const ShmemRecordsSpot *spot = (const ShmemRecordsSpot *) STD_PTR_AT(frame->plan.spots.ptr, i, sizeof(ShmemRecordsSpot)); + ShmemRecordsResolved planned; ShmemRecordsPlaced *placed; count = std_Enumerate_count_step(count); + planned.spot = *spot; + planned.layout = shmem_records_layout(spot); placed = shmem_records_part(SHMEM_RECORDS_PLACED, kind, sizeof(ShmemRecordsPlaced)); placed->spot = *spot; pg_calls_atomic_init_u64(&placed->hand, 0); + pg_calls_atomic_init_u64(&placed->retired, 0); for (size_t index = 0, index_end = (size_t) spot->count; index < index_end; index = std_forward_unchecked_usize_Checked(index, 1)) { - ShmemRecordsMapped record = shmem_records_Mapped_of(spot, index); + ShmemRecordsMapped record = shmem_records_Mapped_of(&planned, index); for (size_t lock = 0, lock_end = (size_t) spot->locks; lock < lock_end; lock = std_forward_unchecked_usize_Checked(lock, 1)) @@ -617,13 +687,7 @@ shmem_records_initialize(const PgMayRaise *raise, ShmemControl *control, int32 t static ShmemRecordsSpot shmem_records_spot(size_t kind) { - ShmemControl *control = shmem_records_control(); - - STD_CHECK_ALIGNED(control, _Alignof(ShmemControl), 321, 24); - STD_CHECK_NULL_DEREF(control, 321, 24); - if (!(kind < (size_t) control->kinds)) - STD_PANIC(321, 9, "a shared record's kind is past the region's kinds"); - return ((ShmemRecordsPlaced *) shmem_records_part(SHMEM_RECORDS_PLACED, kind, sizeof(ShmemRecordsPlaced)))->spot; + return shmem_records_resolved(kind)->spot; } /* shmem_records_kinds returns the number of the region's kinds. */ @@ -769,12 +833,30 @@ shmem_records_hand(size_t kind) return (uint32) STD_REM(u64, hand, (uint64) spot.count, 406, 6); } +/* shmem_records_retired loads an aged kind's retired list's head: its first record plus one, 0 for none. */ +uint64 +shmem_records_retired(size_t kind) +{ + ShmemRecordsPlaced *placed = shmem_records_part(SHMEM_RECORDS_PLACED, kind, sizeof(ShmemRecordsPlaced)); + + return pg_calls_atomic_read_u64(&placed->retired); +} + +/* shmem_records_retired_swap stores an aged kind's retired list's head over the one loaded, reporting whether it was still there. */ +bool +shmem_records_retired_swap(size_t kind, uint64 loaded, uint64 word) +{ + ShmemRecordsPlaced *placed = shmem_records_part(SHMEM_RECORDS_PLACED, kind, sizeof(ShmemRecordsPlaced)); + uint64 seen = loaded; + + pg_calls_atomic_exchange_u64(&placed->retired, &seen, word); + return seen == loaded; +} + static ShmemRecordsMapped shmem_records_record(size_t kind, uint32 index) { - ShmemRecordsSpot spot = shmem_records_spot(kind); - - return shmem_records_Mapped_of(&spot, (size_t) index); + return shmem_records_Mapped_of(shmem_records_resolved(kind), (size_t) index); } /* SHMEM_RECORDS_READERS_LOCK is READERS' RwLock, which guards SHMEM_RECORDS_READERS. */ @@ -821,20 +903,20 @@ shmem_records_clear(size_t kind, uint32 index) { ShmemRecordsMapped record = shmem_records_record(kind, index); - for (size_t atomic = 0, atomic_end = (size_t) record.spot.atomics; atomic < atomic_end; + for (size_t atomic = 0, atomic_end = (size_t) record.spot->atomics; atomic < atomic_end; atomic = std_forward_unchecked_usize_Checked(atomic, 1)) pg_calls_atomic_write_u64(shmem_records_Mapped_atomic(&record, atomic), 0); - for (size_t blob = 0, blob_end = (size_t) record.spot.blobs; blob < blob_end; + for (size_t blob = 0, blob_end = (size_t) record.spot->blobs; blob < blob_end; blob = std_forward_unchecked_usize_Checked(blob, 1)) *shmem_records_Mapped_blob(&record, blob) = 0; - for (size_t array = 0, array_end = (size_t) record.spot.arrays; array < array_end; + for (size_t array = 0, array_end = (size_t) record.spot->arrays; array < array_end; array = std_forward_unchecked_usize_Checked(array, 1)) { uint32 tmp_1; std_index_check(array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 506, 51); std_index_check(array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 506, 80); - tmp_1 = STD_MUL(u32, record.spot.entries[array], record.spot.widths[array], + tmp_1 = STD_MUL(u32, record.spot->entries[array], record.spot->widths[array], 506, 51); std_ptr_write_bytes(shmem_records_Mapped_entries(&record, array), 0, (size_t) tmp_1, 1, 1); } @@ -1065,6 +1147,9 @@ shmem_records_atomic(size_t kind, uint32 index, size_t atomic, ShmemRecordsOp op pg_calls_atomic_exchange_u64(at, &seen, op.u.exchange.f1); return seen; } + case SHMEM_RECORDS_OP_LOAD_ACQUIRE: + pg_calls_read_barrier(); + return pg_calls_atomic_read_u64(at); } pg_unreachable(); } @@ -1097,7 +1182,7 @@ shmem_records_blob_write(size_t kind, uint32 index, size_t blob, const uint8 *by uint64 *length = shmem_records_Mapped_blob(&record, blob); std_index_check(blob, SHMEM_RECORDS_BLOBS, RUST_FILE, 626, 22); - if (bytes_len > (size_t) record.spot.maxes[blob]) + if (bytes_len > (size_t) record.spot->maxes[blob]) { *error = SHMEM_RECORDS_INTERNAL("a shared blob's bytes are longer than its largest size"); return false; @@ -1133,7 +1218,7 @@ shmem_records_entry_read(size_t kind, uint32 index, size_t array, size_t entry, return false; } std_index_check(array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 647, 49); - width = (size_t) record.spot.widths[array]; + width = (size_t) record.spot->widths[array]; std_slice_from_raw_parts_check(at, width, 1, 1, RUST_FILE, 647, 17); std_slice_to_vec_Checked(result, at, width, 1, 1); return true; @@ -1166,7 +1251,7 @@ shmem_records_entry_write(size_t kind, uint32 index, size_t array, size_t entry, return false; } std_index_check(array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 656, 23); - if (bytes_len != (size_t) record.spot.widths[array]) + if (bytes_len != (size_t) record.spot->widths[array]) { *error = SHMEM_RECORDS_INTERNAL("a shared array's entry is of another length than its bytes"); return false; @@ -1255,10 +1340,10 @@ shmem_records_slots_claim(const PgMayRaise *raise, size_t kind, uint32 index, si uint8 *at; std_index_check(array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 713, 24); - if (!(bytes_len == (size_t) record.spot.widths[array])) + if (!(bytes_len == (size_t) record.spot->widths[array])) STD_PANIC(712, 5, "a shared slot's bytes are of another length than its array's entries"); std_index_check(array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 718, 59); - if (pg_calls_atomic_read_u64(count) >= (uint64) record.spot.entries[array]) + if (pg_calls_atomic_read_u64(count) >= (uint64) record.spot->entries[array]) { *result = false; return true; @@ -1320,10 +1405,10 @@ shmem_records_slots_read_body_0(void *frame_) size_t len; std_index_check(frame->array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 739, 50); - width = (size_t) record.spot.widths[frame->array]; + width = (size_t) record.spot->widths[frame->array]; count = pg_calls_atomic_read_u64(shmem_records_Mapped_array(&record, frame->array)); std_index_check(frame->array, SHMEM_RECORDS_ARRAYS, RUST_FILE, 742, 83); - count = std_min_u64(count, (uint64) record.spot.entries[frame->array]); + count = std_min_u64(count, (uint64) record.spot->entries[frame->array]); len = (size_t) count; std_vec_with_capacity_Checked(&frame->entries, len, sizeof(StdVec), _Alignof(StdVec)); frame->entries_live = true; diff --git a/src/doltgres/store/c/collect.c b/src/doltgres/store/c/collect.c index ffb13fc021f0f..27c8f3e2ed907 100644 --- a/src/doltgres/store/c/collect.c +++ b/src/doltgres/store/c/collect.c @@ -30,6 +30,7 @@ #include "doltgres/chunks/file.h" #include "doltgres/chunks/header.h" #include "doltgres/chunks/pack.h" +#include "doltgres/chunks/packs.h" #include "doltgres/chunks/packs/backend.h" #include "doltgres/chunks/packs/marking.h" #include "doltgres/chunks/packs/open.h" @@ -45,6 +46,7 @@ #include "doltgres/std/int128.h" #include "doltgres/std/num.h" #include "doltgres/std/slice.h" +#include "doltgres/std/str.h" #include "doltgres/std/sync.h" #include "doltgres/std/time.h" #include "doltgres/std/tuple.h" @@ -58,7 +60,6 @@ #ifdef DOLTGRES_TEST_STORE #include "doltgres/chunks/address.h" #include "doltgres/simc/simc.h" -#include "doltgres/std/str.h" #include "doltgres/std/test.h" #include "doltgres/store/fixtures.h" #endif @@ -1869,6 +1870,7 @@ typedef struct store_collect_marked_frame_0_t uint64 newest; uint64 limit; uint64 percent; + uint32 emptied; bool *has_result; StoreCollectCollected *result; HostError *error; @@ -2131,6 +2133,8 @@ store_collect_marked_choose(store_collect_marked_frame_0_t *frame) live = store_collect_bytes_get(&frame->marks.bytes, entry.f0); if (live == 0) std_vec_push_Checked(&frame->dead, &entry.f0, sizeof(uint64), _Alignof(uint64)); + else if (entry.f1 == frame->emptied && frame->emptied != CHUNKS_PACKS_DEFAULT_SPACE) + store_collect_copied_push(&frame->copied, CHUNKS_PACKS_DEFAULT_SPACE, entry.f0); else if (store_collect_over(chunks_pack_PackReader_records_len(std_arc_Deref_deref(reader, _Alignof(ChunksPackPackReader))), live, frame->percent)) @@ -2387,12 +2391,12 @@ store_collect_marked_body_0(void *frame_) */ static bool store_collect_marked(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, uint32 database, - const StdVec *old, uint64 newest, uint64 limit, uint64 percent, bool *has_result, + const StdVec *old, uint64 newest, uint64 limit, uint64 percent, uint32 emptied, bool *has_result, StoreCollectCollected *result, HostError *error) { store_collect_marked_frame_0_t frame = {.io = io, .repo = repo, .reading = reading, .database = database, - .old = old, .newest = newest, .limit = limit, .percent = percent, .has_result = has_result, .result = result, - .error = error}; + .old = old, .newest = newest, .limit = limit, .percent = percent, .emptied = emptied, .has_result = has_result, + .result = result, .error = error}; std_landing(&frame, store_collect_marked_body_0, store_collect_marked_pad_0); return frame.ok; @@ -2406,7 +2410,7 @@ store_collect_marked(HostIoIo *io, const RepoRepo *repo, const HostSharedReader */ static bool store_collect_marking(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, uint32 database, - const StdVec *old, uint64 limit, uint64 percent, bool *has_result, + const StdVec *old, uint64 limit, uint64 percent, uint32 emptied, bool *has_result, StoreCollectCollected *result, HostError *error) { const ChunksPacksBackendPacks *packs = &repo->packs; @@ -2421,7 +2425,7 @@ store_collect_marking(HostIoIo *io, const RepoRepo *repo, const HostSharedReader if (!chunks_packs_marking_Packs_begin_marking(packs, io, reading, &newest, &failed)) goto raised; - if (store_collect_marked(io, repo, reading, database, old, newest, limit, percent, has_result, result, + if (store_collect_marked(io, repo, reading, database, old, newest, limit, percent, emptied, has_result, result, &collect_error)) { if (*has_result) @@ -2451,11 +2455,14 @@ store_collect_marking(HostIoIo *io, const RepoRepo *repo, const HostSharedReader * base backup is counted in. Its marking's reads and its copies run at limit * bytes a second, or unpaced at 0, and it copies out the live chunks of a * pack that passes them by percent. It answers nothing when a base backup - * kept it from finishing. + * kept it from finishing. A tablespace it empties, unless that is + * CHUNKS_PACKS_DEFAULT_SPACE, has the live chunks of each of its packs copied + * into the repository's own directory whatever their share, so that the + * packs retire, as DROP TABLESPACE needs (bugs row 99). */ bool store_collect_collected(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, uint32 database, - uint64 limit, uint64 percent, const StdVec *old, bool *has_result, + uint64 limit, uint64 percent, uint32 emptied, const StdVec *old, bool *has_result, StoreCollectCollected *result, HostError *error) { bool done; @@ -2465,7 +2472,8 @@ store_collect_collected(HostIoIo *io, const RepoRepo *repo, const HostSharedRead return false; if (!host_collection_begin()) return true; - done = store_collect_marking(io, repo, reading, database, old, limit, percent, has_result, result, error); + done = store_collect_marking(io, repo, reading, database, old, limit, percent, emptied, has_result, result, + error); host_collection_end(); return done; } @@ -2710,8 +2718,8 @@ store_collect_checked(HostIoIo *io, uint32 repository, const StoreCollectDatabas old = held->old; std_vec_new(&held->old, sizeof(uint64), _Alignof(uint64)); } - if (!store_collect_collected(io, &repo, reading, database->id, limit, settings->target_percent, &old, &has_done, - &done, error)) + if (!store_collect_collected(io, &repo, reading, database->id, limit, settings->target_percent, + CHUNKS_PACKS_DEFAULT_SPACE, &old, &has_done, &done, error)) { std_vec_free_Checked(&old, sizeof(uint64), _Alignof(uint64)); goto dropped; @@ -2927,9 +2935,88 @@ store_collect_collect_due(HostIoIo *io, StoreCollectSchedule *schedule, const St return frame.ok; } -/* store_collect_asked is an ask's full collection under its database's lock, the repository opened for it and closed after; its caller gives the lock back however it ends. */ +/* + * store_collect_unlisted takes an emptied tablespace out of a repository's + * lists once the repository lists none of its packs (bugs row 99). + */ +static bool +store_collect_unlisted(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, uint32 emptied, + HostError *error) +{ + ChunksErrorError failed; + StdVec spaces; + bool listed = false; + + if (!chunks_packs_backend_Packs_spaces(&repo->packs, io, reading, &spaces, &failed)) + { + *error = chunks_packs_shared_raised(&failed); + return false; + } + for (size_t k = 0, k_len = spaces.len; k < k_len; k++) + listed |= STD_VEC_AT(&spaces, std_TupleU64U32_t, k).f1 == emptied; + std_vec_free_Checked(&spaces, sizeof(std_TupleU64U32_t), _Alignof(std_TupleU64U32_t)); + if (listed) + return true; + if (!chunks_packs_backend_Packs_remove_space(&repo->packs, HOST_IO_IO_TAKES_SHORT(io), reading, emptied, &failed)) + { + *error = chunks_packs_shared_raised(&failed); + return false; + } + return true; +} + +/* + * store_collect_emptied removes the files due removal, and then a + * repository's directory in an emptied tablespace once no pack file is left + * in it, every one either copied out and retired with its readers ended or + * never listed; a pack file still there, a pack being written included, + * leaves the directory for a later ask. A writer that opens in the + * tablespace after the directory's listing finds it gone and its pack's + * write fails, which leaves its fold's delta for a retry that makes the + * directory again, and its relation's own file there makes Postgres refuse + * the DROP. + */ static bool -store_collect_asked(HostIoIo *io, uint32 repository, const StoreCollectDatabase *database, +store_collect_emptied(HostIoIo *io, uint32 database, uint32 emptied, HostError *error) +{ + ChunksErrorError failed; + StdPathBuf dir; + StdVec names; + bool has; + bool packed = false; + bool removed; + bool ok = true; + + if (!chunks_packs_removal_remove_due(io, &failed)) + { + *error = chunks_packs_shared_raised(&failed); + return false; + } + dir = host_tablespace_dir(database, emptied); + if (!host_list_dir_if_there(HOST_IO_IO_READ_DOORS(io), (const char *) dir.bytes.ptr, dir.bytes.len, &has, &names, + error)) + ok = false; + else if (has) + { + for (size_t k = 0, k_len = names.len; k < k_len; k++) + { + const StdString *name = &STD_VEC_AT(&names, StdString, k); + + packed |= (name->len >= 5 && memcmp((const char *) name->ptr + name->len - 5, ".pack", 5) == 0) || + (name->len >= 9 && memcmp((const char *) name->ptr + name->len - 9, ".pack.tmp", 9) == 0); + } + std_vec_drop_Checked(&names, sizeof(StdString), _Alignof(StdString), std_String_drop); + if (!packed) + ok = host_remove_tree(HOST_IO_IO_IO_DOORS(io), (const char *) dir.bytes.ptr, dir.bytes.len, &removed, + error); + } + std_vec_free_Checked(&dir.bytes, 1, 1); + return ok; +} + +/* store_collect_asked is an ask's full collection under its database's lock, emptying the tablespace it names, the repository opened for it and closed after; its caller gives the lock back however it ends. */ +static bool +store_collect_asked(HostIoIo *io, uint32 repository, const StoreCollectDatabase *database, uint32 emptied, const StoreCollectSettings *settings, bool *has_result, StoreCollectCollected *result, HostError *error) { @@ -2947,8 +3034,14 @@ store_collect_asked(HostIoIo *io, uint32 repository, const StoreCollectDatabase return false; } std_vec_new(&none, sizeof(uint64), _Alignof(uint64)); - ok = store_collect_collected(io, &repo, &reading, database->id, 0, settings->target_percent, &none, has_result, - result, error); + ok = store_collect_collected(io, &repo, &reading, database->id, 0, settings->target_percent, emptied, &none, + has_result, result, error); + if (ok && *has_result && emptied != CHUNKS_PACKS_DEFAULT_SPACE && + !store_collect_unlisted(io, &repo, &reading, emptied, error)) + { + store_collect_Collected_drop(result); + ok = false; + } if (!ok) repo_Repo_drop(&repo); else if (!repo_Repo_close(&repo, io, &reading, &failed)) @@ -2959,12 +3052,18 @@ store_collect_asked(HostIoIo *io, uint32 repository, const StoreCollectDatabase ok = false; } host_shared_Reader_drop(&reading); + if (ok && *has_result && emptied != CHUNKS_PACKS_DEFAULT_SPACE && !store_collect_emptied(io, repository, emptied, error)) + { + store_collect_Collected_drop(result); + ok = false; + } return ok; } /* * store_collect_collect_asked runs the full collection a backend asked of the - * collector worker in a database's repository, recorded in the schedule as + * collector worker in a database's repository, emptying the tablespace the + * ask names (collected), recorded in the schedule as * its own, under the database's lock as collect_due takes it, and returns its * counts, a [u64; 3]: the packs it retired, those it copied out, and its live * bytes, all 0 when the database may not be collected, a session holds its @@ -2972,8 +3071,8 @@ store_collect_asked(HostIoIo *io, uint32 repository, const StoreCollectDatabase */ bool store_collect_collect_asked(HostIoIo *io, StoreCollectSchedule *schedule, const StoreCollectDatabase *databases, - size_t databases_len, uint32 id, const StoreCollectSettings *settings, uint64 *result, - HostError *error) + size_t databases_len, uint32 id, uint32 emptied, const StoreCollectSettings *settings, + uint64 *result, HostError *error) { const StoreCollectDatabase *database = store_collect_find(databases, databases_len, id, true); StoreCollectCollected done; @@ -2996,7 +3095,7 @@ store_collect_collect_asked(HostIoIo *io, StoreCollectSchedule *schedule, const return false; if (!locked) return true; - ok = store_collect_asked(io, id, database, settings, &has, &done, error); + ok = store_collect_asked(io, id, database, emptied, settings, &has, &done, error); host_database_unlocked(id); if (!ok) return false; @@ -3238,8 +3337,8 @@ store_collect_tests_collected(HostIoIo *io, const RepoRepo *repo, const HostShar bool has; std_vec_new(&none, sizeof(uint64), _Alignof(uint64)); - store_collect_tests_ok(store_collect_collected(io, repo, reading, database, limit, 50, old == NULL ? &none : old, - &has, &done, &error), &error, line, col); + store_collect_tests_ok(store_collect_collected(io, repo, reading, database, limit, 50, CHUNKS_PACKS_DEFAULT_SPACE, + old == NULL ? &none : old, &has, &done, &error), &error, line, col); if (!has) std_option_unwrap_failed(RUST_FILE, line, some_col); return done; @@ -3408,8 +3507,8 @@ test_store_collect_tests_a_collection_runs_nothing_while_a_base_backup_is_counte store_collect_tests_ok(host_backup_begin(io, &error), &error, 924, 32); STD_ASSERT(925, 9, host_backups_running(), "host::backups_running()"); std_vec_new(&none, sizeof(uint64), _Alignof(uint64)); - store_collect_tests_ok(store_collect_collected(io, &repo, &reading, 832, 0, 50, &none, &has, &collected, &error), - &error, 926, 85); + store_collect_tests_ok(store_collect_collected(io, &repo, &reading, 832, 0, 50, CHUNKS_PACKS_DEFAULT_SPACE, &none, + &has, &collected, &error), &error, 926, 85); if (!store_collect_tests_option_eq(has, &collected, false, NULL)) { StdOptionRef left = {has, STD_ITEM(&collected, store_collect_Collected_Debug_fmt)}; diff --git a/src/doltgres/test/meson.build b/src/doltgres/test/meson.build index 22c508c2ab66d..22833df2c5362 100644 --- a/src/doltgres/test/meson.build +++ b/src/doltgres/test/meson.build @@ -135,6 +135,14 @@ tests += { ] : []) + [ 't/123_cport_fe_pg.pl', 't/120_cport_access.pl', + 't/071_spill_reads.pl', + 't/131_scan_costs.pl', + 't/132_heap_only_contrib.pl', + 't/072_standby_spills.pl', + 't/124_carry_unlisted.pl', + 't/135_moved_takes.pl', + 't/133_identity_spill.pl', + 't/134_saves.pl', ], 'deps': [doltgres_cport_test_fe_pg], # The injection points are cluster-wide, so disable installcheck diff --git a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_chunks.c b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_chunks.c index 489c006381360..b76a2c0003891 100644 --- a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_chunks.c +++ b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_chunks.c @@ -227,6 +227,8 @@ static const CportTest CPORT_TESTS[] = { test_chunks_packs_lists_tests_a_pack_added_is_listed_by_every_view}, {"packs::lists::tests::a_refresh_across_a_whole_write_lists_the_packs_before_it_or_after_it", test_chunks_packs_lists_tests_a_refresh_across_a_whole_write_lists_the_packs_before_it_or_after_it}, + {"packs::lists::tests::a_removed_tablespace_reads_a_runs_file_made_again_from_its_start", + test_chunks_packs_lists_tests_a_removed_tablespace_reads_a_runs_file_made_again_from_its_start}, {"packs::lists::tests::a_retirements_records_lead_every_view_to_the_copies", test_chunks_packs_lists_tests_a_retirements_records_lead_every_view_to_the_copies}, {"packs::lists::tests::a_whole_write_that_fails_before_its_renames_leaves_the_count_even", @@ -408,6 +410,8 @@ static const CportTest CPORT_TESTS[] = { {"trailer::tests::internal_nodes_store_their_subtrees_highest_tuple_ids", test_chunks_trailer_tests_internal_nodes_store_their_subtrees_highest_tuple_ids}, {"trailer::tests::round_trips", test_chunks_trailer_tests_round_trips}, + {"trailer::tests::stamp_of_finds_each_copys_stamp_among_its_runs", + test_chunks_trailer_tests_stamp_of_finds_each_copys_stamp_among_its_runs}, {"trailer::tests::the_same_stamps_store_the_same_bytes_however_they_were_pushed", test_chunks_trailer_tests_the_same_stamps_store_the_same_bytes_however_they_were_pushed}, {"trailer::tests::xxh3_digests_are_the_committed_table", diff --git a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_doltgres.c b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_doltgres.c index aef06e33606fb..7790c079731cb 100644 --- a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_doltgres.c +++ b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_doltgres.c @@ -118,8 +118,12 @@ static const CportTest CPORT_TESTS[] = { test_doltgres_scan_tests_a_parallel_tid_ranges_blocks_are_heaps}, {"scan::tests::a_sample_of_many_blocks_walks_the_delta_once", test_doltgres_scan_tests_a_sample_of_many_blocks_walks_the_delta_once}, + {"scan::tests::a_scan_held_inside_its_delta_read_across_an_install_and_drop_gives_every_row_once", + test_doltgres_scan_tests_a_scan_held_inside_its_delta_read_across_an_install_and_drop_gives_every_row_once}, {"scan::tests::a_scan_merges_the_deltas_rows_with_the_bases_in_key_order", test_doltgres_scan_tests_a_scan_merges_the_deltas_rows_with_the_bases_in_key_order}, + {"scan::tests::a_scan_moves_to_a_newer_base_at_its_next_read_and_gives_every_row_once", + test_doltgres_scan_tests_a_scan_moves_to_a_newer_base_at_its_next_read_and_gives_every_row_once}, {"scan::tests::a_scan_reads_a_bases_rows_in_order_across_calls", test_doltgres_scan_tests_a_scan_reads_a_bases_rows_in_order_across_calls}, {"scan::tests::a_scan_reads_backward_and_turns_as_heaps_does", @@ -136,6 +140,8 @@ static const CportTest CPORT_TESTS[] = { test_doltgres_scan_tests_a_tuple_id_is_valid_within_the_storages_blocks}, {"scan::tests::each_keyless_copy_fetches_by_its_own_tuple_id", test_doltgres_scan_tests_each_keyless_copy_fetches_by_its_own_tuple_id}, + {"scan::tests::fetches_through_one_memo_give_each_keyless_copys_own_stamp", + test_doltgres_scan_tests_fetches_through_one_memo_give_each_keyless_copys_own_stamp}, {"scan::tests::scans_of_one_base_keep_their_own_places", test_doltgres_scan_tests_scans_of_one_base_keep_their_own_places}, {"sizes::tests::a_tables_blocks_are_its_rows_packed_and_at_least_its_ctid_blocks", diff --git a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_host.c b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_host.c index 121f8eac45146..24fbd5b436a23 100644 --- a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_host.c +++ b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_host.c @@ -82,6 +82,10 @@ static const CportTest CPORT_TESTS[] = { test_std_vec_tests_an_over_aligned_element_stays_aligned}, {"delta::blocks::tests::a_body_takes_the_smallest_class_that_holds_it", test_host_delta_blocks_tests_a_body_takes_the_smallest_class_that_holds_it}, + {"delta::directory::tests::a_free_pass_loads_only_the_retirements", + test_host_delta_directory_tests_a_free_pass_loads_only_the_retirements}, + {"delta::directory::tests::a_refused_allocations_pass_loads_only_the_retirements", + test_host_delta_directory_tests_a_refused_allocations_pass_loads_only_the_retirements}, {"delta::directory::tests::a_size_fits_its_word", test_host_delta_directory_tests_a_size_fits_its_word}, {"delta::directory::tests::a_words_index_is_below_its_pages_count", diff --git a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_simc.c b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_simc.c index ec362f61e02e1..18b1876c17c57 100644 --- a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_simc.c +++ b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_simc.c @@ -2533,6 +2533,68 @@ test_simc_tests_a_scripted_run_runs_each_actor_until_its_point_and_terminates_on simc_TestIo_drop(&test_io); } +/* test_simc_tests_trace_Debug_fmt is the trace case's Vec Debug. */ +static bool +test_simc_tests_trace_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *trace = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < trace->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(trace, size_t, k), std_usize_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* + * test_simc_tests_a_reports_trace_is_the_rusts: + * a finished run's trace is its report's, each yield point's actor in order + * as the Rust's Debug of the report shows it, the first actor's pause first. + */ +static void +test_simc_tests_a_reports_trace_is_the_rusts(void) +{ + static const StdFmtPiece report_pieces[] = {STD_FMT_ARG(0)}; + static const StdFmtPiece trace_pieces[] = {STD_FMT_LIT("trace: "), STD_FMT_ARG(0), STD_FMT_LIT(", exits")}; + SimcTestIo test_io = simc_io(); + SimcRun run = simc_Run_new(13); + SimcActor first = {test_simc_tests_pausing_actor, (void *) (intptr_t) 1}; + SimcActor second = {test_simc_tests_pausing_actor, (void *) (intptr_t) 2}; + SimcStep steps[] = {{"second", 6, PG_UINT64_MAX}, {"first", 5, PG_UINT64_MAX}}; + SimcReport report; + StdVec trace; + StdString text; + StdString shown; + StdFmtArg report_args[1]; + StdFmtArg trace_args[1]; + StdFmtArguments report_formatted = {report_pieces, lengthof(report_pieces), report_args}; + StdFmtArguments trace_formatted = {trace_pieces, lengthof(trace_pieces), trace_args}; + bool found = false; + + test_simc_tests_FINISHED = 0; + simc_Run_actor(&run, "first", 5, first); + simc_Run_actor(&run, "second", 6, second); + simc_Run_until(&run, "first", 5, "doltgres-simc-case-first"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + trace = simc_Report_trace(&report); + report_args[0].value = &report; + report_args[0].fmt = simc_Report_Debug_fmt; + std_fmt_format_Checked(&text, &report_formatted); + trace_args[0].value = &trace; + trace_args[0].fmt = test_simc_tests_trace_Debug_fmt; + std_fmt_format_Checked(&shown, &trace_formatted); + for (size_t at = 0; !found && at + shown.len <= text.len; at++) + found = memcmp((const uint8 *) text.ptr + at, shown.ptr, shown.len) == 0; + STD_ASSERT(__LINE__, 5, trace.len >= 2 && STD_VEC_AT(&trace, size_t, 0) == 0, + "the first actor's pause is the trace's first yield"); + STD_ASSERT(__LINE__, 5, found, "the trace is the report's"); + std_vec_free_Checked(&shown, 1, 1); + std_vec_free_Checked(&text, 1, 1); + std_vec_free_Checked(&trace, sizeof(size_t), _Alignof(size_t)); + simc_Report_drop(&report); + simc_TestIo_drop(&test_io); +} + /* test_simc_tests_acting is the acting case's body: it records that it ran. */ static void test_simc_tests_acting(void *captures) @@ -3307,6 +3369,211 @@ test_simc_tests_the_checkpoint_delay_doors_answer_as_the_simulations_host(void) simc_TestIo_drop(&test_io); } +/* test_simc_tests_horizon is the horizon case's actor: a NonVacuumable snapshot's removable xids. */ +static void +test_simc_tests_horizon(void *captures) +{ + HostHost host = host_current(); + HostMayRaise raise = host_doors(HOST_IO_IO_READ_DOORS((HostIoIo *) captures)); + HostXid horizon = {100}; + HostXid below = {99}; + SimcTaken taken = simc_non_vacuumable(horizon); + HostSnapshot snapshot = simc_Taken_lent(&taken); + HostError error; + bool removable; + + STD_ASSERT(__LINE__, 5, snapshot.kind == HOST_SNAPSHOT_KIND_NON_VACUUMABLE, "the snapshot is NonVacuumable"); + STD_ASSERT(__LINE__, 5, host.ops->removable_in(host.self, &raise, snapshot.of, below, &removable, &error) && + removable, "an xid below the horizon is removable"); + STD_ASSERT(__LINE__, 5, host.ops->removable_in(host.self, &raise, snapshot.of, horizon, &removable, &error) && + !removable, "the horizon's own xid isn't"); + simc_Taken_drop(&taken); +} + +/* + * test_simc_tests_a_non_vacuumable_snapshot_finds_xids_below_its_horizon_removable: + * a NonVacuumable snapshot is of its kind, and through the removable_in door + * it finds an xid below its horizon removable and the horizon's own not. + */ +static void +test_simc_tests_a_non_vacuumable_snapshot_finds_xids_below_its_horizon_removable(void) +{ + SimcTestIo test_io = simc_io(); + + simc_as_actor(55, test_simc_tests_horizon, &test_io.io); + simc_TestIo_drop(&test_io); +} + +/* test_simc_tests_logging is the logged case's actor: it inserts a record and reads it back as the WAL took it. */ +static void +test_simc_tests_logging(void *captures) +{ + static const uint8 body[] = {1, 2, 3}; + HostHost host = host_current(); + HostMayRaise raise = host_doors(HOST_IO_IO_READ_DOORS((HostIoIo *) captures)); + HostError error; + HostLsn end; + StdVec logged; + const simc_TupleU8VecU8Lsn_t *last; + + STD_ASSERT(__LINE__, 5, host.ops->wal_insert(host.self, &raise, HOST_WAL_KIND_CHUNK, body, sizeof(body), &end, + &error), "the record is inserted"); + logged = simc_logged(); + STD_ASSERT(__LINE__, 5, logged.len > 0, "the WAL took a record"); + last = &STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, logged.len - 1); + STD_ASSERT(__LINE__, 5, last->f0 == host_WalKind_value(HOST_WAL_KIND_CHUNK) && last->f1.len == sizeof(body) && + memcmp(last->f1.ptr, body, sizeof(body)) == 0 && last->f2 == end, + "the last record logged is the record, its kind, and its end"); + for (size_t i = 0; i < logged.len; i++) + std_vec_free_Checked(&STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, i).f1, 1, 1); + std_vec_free_Checked(&logged, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); +} + +/* + * test_simc_tests_the_logged_records_are_the_records_the_wal_took: the + * records simc lists are those the WAL door took, each with its kind, its + * body, and its end. + */ +static void +test_simc_tests_the_logged_records_are_the_records_the_wal_took(void) +{ + SimcTestIo test_io = simc_io(); + + simc_as_actor(56, test_simc_tests_logging, &test_io.io); + simc_TestIo_drop(&test_io); +} + +/* + * test_simc_tests_a_delta_page_takes_one_of_the_directorys_free_slots: a page + * the delta takes holds one of its directory's free slots until it's given + * back. + */ +static void +test_simc_tests_a_delta_page_takes_one_of_the_directorys_free_slots(void) +{ + SimcTestIo test_io = simc_io(); + HostHost host = host_current(); + HostMayRaise raise = host_doors(HOST_IO_IO_READ_DOORS(&test_io.io)); + HostDeltaPageSize size = {.bytes = 512, .words = 4}; + HostError error; + StdVec names; + bool refused; + uint64 frees; + size_t before = simc_free_slots(); + + std_vec_with_capacity_Checked(&names, 1, sizeof(HostDeltaPagePage), _Alignof(HostDeltaPagePage)); + STD_ASSERT(__LINE__, 5, host.ops->delta_alloc(host.self, &raise, &size, 1, &names, &refused, &frees, &error) && + !refused, "the page is taken"); + STD_ASSERT(__LINE__, 5, simc_free_slots() == before - 1, "the page holds one free slot"); + host.ops->delta_give_back(host.self, STD_VEC_AT(&names, HostDeltaPagePage, 0)); + std_vec_free_Checked(&names, sizeof(HostDeltaPagePage), _Alignof(HostDeltaPagePage)); + host.ops->delta_pass(host.self); + simc_TestIo_drop(&test_io); +} + +/* test_simc_tests_new_ciding is the new cid case's actor: it logs a row's command ids and reads the record back. */ +static void +test_simc_tests_new_ciding(void *captures) +{ + HostHost host = host_current(); + HostMayRaise raise = host_doors(HOST_IO_IO_READ_DOORS((HostIoIo *) captures)); + HostStorage storage = {HOST_PG_DEFAULT, 9301, 16384}; + HostRow row = {UINT64CONST(1) << 16 | 2}; + HostNewCid cid = {.has_cmin = true, .cmin = {3}, .has_combo = true, .combo = {5}}; + HostError error; + HostLsn end; + StdVec cids; + const simc_TupleStorageRowNewCidLsn_t *last; + + STD_ASSERT(__LINE__, 5, host.ops->wal_new_cid(host.self, &raise, &storage, row, cid, &end, &error), + "the new cid record is logged"); + cids = simc_new_cids(); + STD_ASSERT(__LINE__, 5, cids.len > 0, "the WAL took a new cid record"); + last = &STD_VEC_AT(&cids, simc_TupleStorageRowNewCidLsn_t, cids.len - 1); + STD_ASSERT(__LINE__, 5, last->f0.database == 9301 && last->f0.relation == 16384 && last->f1.f0 == row.f0 && + last->f2.has_cmin && last->f2.cmin.f0 == 3 && !last->f2.has_cmax && last->f2.has_combo && + last->f2.combo.f0 == 5 && last->f3 == end, "the last record is the row's, its cids, and its end"); + std_vec_free_Checked(&cids, sizeof(simc_TupleStorageRowNewCidLsn_t), _Alignof(simc_TupleStorageRowNewCidLsn_t)); +} + +/* + * test_simc_tests_the_new_cid_door_logs_its_record_as_the_simulations_host: + * the wal_new_cid door logs a row's command ids with its storage and row, + * which simc lists with the record's end. + */ +static void +test_simc_tests_the_new_cid_door_logs_its_record_as_the_simulations_host(void) +{ + SimcTestIo test_io = simc_io(); + + simc_as_actor(57, test_simc_tests_new_ciding, &test_io.io); + simc_TestIo_drop(&test_io); +} + +/* + * test_simc_tests_the_wal_flush_door_answers_as_the_simulations_host: a + * flush of the WAL up to a record's end answers, as the simulation's host + * flushes it. + */ +static void +test_simc_tests_the_wal_flush_door_answers_as_the_simulations_host(void) +{ + static const uint8 body[] = {1, 2, 3}; + SimcTestIo test_io = simc_io(); + HostHost host = host_current(); + HostMayRaise raise = host_doors(HOST_IO_IO_READ_DOORS(&test_io.io)); + HostError error; + HostLsn end; + + STD_ASSERT(__LINE__, 5, host.ops->wal_insert(host.self, &raise, HOST_WAL_KIND_CHUNK, body, sizeof(body), &end, + &error), "the record is inserted"); + STD_ASSERT(__LINE__, 5, host.ops->wal_flush(host.self, &raise, end, &error), "the WAL is flushed to its end"); + simc_TestIo_drop(&test_io); +} + +/* + * test_simc_tests_the_injected_door_answers_as_the_simulations_host: the + * simulation attaches no injection point, so the injected door answers false + * for a point's name. + */ +static void +test_simc_tests_the_injected_door_answers_as_the_simulations_host(void) +{ + SimcTestIo test_io = simc_io(); + HostHost host = host_current(); + HostMayRaise raise = host_doors(HOST_IO_IO_READ_DOORS(&test_io.io)); + HostError error; + bool injected = true; + + STD_ASSERT(__LINE__, 5, host.ops->injected(host.self, &raise, "doltgres-vacuum-found", &injected, &error) && + !injected, "the simulation attaches no injection point"); + simc_TestIo_drop(&test_io); +} + +/* + * test_simc_tests_a_hold_of_the_test_process_keeps_its_data_directory_past_its_capability: + * a hold of the test process taken before a test's capability keeps the + * test's data directory once the capability drops, and its release ends the + * test. + */ +static void +test_simc_tests_a_hold_of_the_test_process_keeps_its_data_directory_past_its_capability(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + StdPathBuf before = simc_data_dir(); + StdPathBuf after; + + simc_TestIo_drop(&test_io); + after = simc_data_dir(); + STD_ASSERT(__LINE__, 5, before.bytes.len == after.bytes.len && + memcmp(before.bytes.ptr, after.bytes.ptr, before.bytes.len) == 0, + "the data directory outlives the capability"); + std_vec_free_Checked(&after.bytes, 1, 1); + std_vec_free_Checked(&before.bytes, 1, 1); + simc_Process_drop(&process); +} + static const CportTest CPORT_TESTS[] = { {"simc::tests::a_body_as_an_actor_runs_with_that_actors_id_in_the_data_directory", test_simc_tests_a_body_as_an_actor_runs_with_that_actors_id_in_the_data_directory}, @@ -3322,6 +3589,8 @@ static const CportTest CPORT_TESTS[] = { test_simc_tests_a_crash_fails_each_call_until_a_recovery_takes_the_files_back_to_their_syncs}, {"simc::tests::a_delta_page_is_taken_written_and_given_back_as_the_simulations_host", test_simc_tests_a_delta_page_is_taken_written_and_given_back_as_the_simulations_host}, + {"simc::tests::a_delta_page_takes_one_of_the_directorys_free_slots", + test_simc_tests_a_delta_page_takes_one_of_the_directorys_free_slots}, {"simc::tests::a_dictionary_simc_prepares_compresses_and_decompresses_and_drops_in_the_c", test_simc_tests_a_dictionary_simc_prepares_compresses_and_decompresses_and_drops_in_the_c}, {"simc::tests::a_failed_allocation_of_a_listings_names_frees_the_names_made_and_raises_out_of_memory", @@ -3336,10 +3605,14 @@ static const CportTest CPORT_TESTS[] = { test_simc_tests_a_hinted_c_error_crosses_to_sim_as_its_literals}, {"simc::tests::a_historic_snapshot_lends_its_bounds_command_and_sorted_xids", test_simc_tests_a_historic_snapshot_lends_its_bounds_command_and_sorted_xids}, + {"simc::tests::a_hold_of_the_test_process_keeps_its_data_directory_past_its_capability", + test_simc_tests_a_hold_of_the_test_process_keeps_its_data_directory_past_its_capability}, {"simc::tests::a_linked_delta_page_opens_reads_appends_seals_and_retires_as_the_simulations_host", test_simc_tests_a_linked_delta_page_opens_reads_appends_seals_and_retires_as_the_simulations_host}, {"simc::tests::a_nap_is_counted_and_the_disk_space_set_is_answered", test_simc_tests_a_nap_is_counted_and_the_disk_space_set_is_answered}, + {"simc::tests::a_non_vacuumable_snapshot_finds_xids_below_its_horizon_removable", + test_simc_tests_a_non_vacuumable_snapshot_finds_xids_below_its_horizon_removable}, {"simc::tests::a_pause_a_warning_a_log_and_the_last_redo_answer_as_the_simulations_host", test_simc_tests_a_pause_a_warning_a_log_and_the_last_redo_answer_as_the_simulations_host}, {"simc::tests::a_pooled_records_doors_answer_as_the_simulations_host", @@ -3356,6 +3629,8 @@ static const CportTest CPORT_TESTS[] = { test_simc_tests_a_refused_take_fails_once_and_refuse_nothing_takes_back_the_rest}, {"simc::tests::a_reports_debug_is_the_rusts", test_simc_tests_a_reports_debug_is_the_rusts}, + {"simc::tests::a_reports_trace_is_the_rusts", + test_simc_tests_a_reports_trace_is_the_rusts}, {"simc::tests::a_returning_actor_ends_ok_and_its_endings_compare_as_the_rusts", test_simc_tests_a_returning_actor_ends_ok_and_its_endings_compare_as_the_rusts}, {"simc::tests::a_run_check_fails_the_run_at_a_yield_with_its_message", @@ -3426,8 +3701,14 @@ static const CportTest CPORT_TESTS[] = { test_simc_tests_the_delta_hold_and_budget_doors_answer_as_the_simulations_host}, {"simc::tests::the_epoch_doors_answer_as_the_simulations_host", test_simc_tests_the_epoch_doors_answer_as_the_simulations_host}, + {"simc::tests::the_injected_door_answers_as_the_simulations_host", + test_simc_tests_the_injected_door_answers_as_the_simulations_host}, {"simc::tests::the_key_doors_answer_as_the_simulations_host", test_simc_tests_the_key_doors_answer_as_the_simulations_host}, + {"simc::tests::the_logged_records_are_the_records_the_wal_took", + test_simc_tests_the_logged_records_are_the_records_the_wal_took}, + {"simc::tests::the_new_cid_door_logs_its_record_as_the_simulations_host", + test_simc_tests_the_new_cid_door_logs_its_record_as_the_simulations_host}, {"simc::tests::the_next_decompression_runs_out_of_memory_when_asked", test_simc_tests_the_next_decompression_runs_out_of_memory_when_asked}, {"simc::tests::the_node_cache_installs_finds_and_unpins_a_nodes_frame", @@ -3444,6 +3725,8 @@ static const CportTest CPORT_TESTS[] = { test_simc_tests_the_test_only_std_fs_writes_reads_appends_and_removes_a_file}, {"simc::tests::the_wal_a_checkpoint_and_a_databases_quiet_answer_as_the_simulations_host", test_simc_tests_the_wal_a_checkpoint_and_a_databases_quiet_answer_as_the_simulations_host}, + {"simc::tests::the_wal_flush_door_answers_as_the_simulations_host", + test_simc_tests_the_wal_flush_door_answers_as_the_simulations_host}, }; PG_FUNCTION_INFO_V1(doltgres_cport_test_simc); diff --git a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_txn.c b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_txn.c index 1079d9d6ab9d6..4dd6268d6effd 100644 --- a/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_txn.c +++ b/src/doltgres/test/modules/doltgres_cport_test/doltgres_cport_test_txn.c @@ -25,17 +25,25 @@ #include "fmgr.h" #include "doltgres/txn/base.h" +#include "doltgres/txn/blobs.h" +#include "doltgres/txn/change.h" #include "doltgres/txn/counted.h" +#include "doltgres/txn/entry.h" #include "doltgres/txn/free_space.h" +#include "doltgres/txn/identity.h" #include "doltgres/txn/memtable/branch.h" #include "doltgres/txn/memtable/index.h" #include "doltgres/txn/memtable/map.h" #include "doltgres/txn/memtable/table.h" #include "doltgres/txn/memtable/tuples.h" +#include "doltgres/txn/outcome.h" +#include "doltgres/txn/record.h" #include "doltgres/txn/runs.h" #include "doltgres/txn/spill.h" #include "doltgres/txn/state.h" #include "doltgres/txn/status.h" +#include "doltgres/txn/vacuum.h" +#include "doltgres/txn/values.h" #include "doltgres/txn/visible.h" #include "doltgres_cport_test.h" @@ -57,6 +65,24 @@ static const CportTest CPORT_TESTS[] = { test_txn_base_tests_planned_counts_follow_the_sizes_alone}, {"base::tests::planned_counts_of_entries_larger_than_a_node_are_those_of_the_tree_the_splitter_makes", test_txn_base_tests_planned_counts_of_entries_larger_than_a_node_are_those_of_the_tree_the_splitter_makes}, + {"blobs::tests::a_blob_in_the_delta_reads_back_through_blobs", + test_txn_blobs_tests_a_blob_in_the_delta_reads_back_through_blobs}, + {"blobs::tests::a_chunk_another_running_transaction_holds_stays_after_that_ones_abort", + test_txn_blobs_tests_a_chunk_another_running_transaction_holds_stays_after_that_ones_abort}, + {"blobs::tests::a_logged_write_logs_a_chunk_an_unlogged_one_left_and_redo_makes_it_durable", + test_txn_blobs_tests_a_logged_write_logs_a_chunk_an_unlogged_one_left_and_redo_makes_it_durable}, + {"blobs::tests::a_rolled_back_chunk_takes_nothing_from_its_tops_count", + test_txn_blobs_tests_a_rolled_back_chunk_takes_nothing_from_its_tops_count}, + {"blobs::tests::a_transactions_chunks_spill_read_back_and_leave_at_its_abort", + test_txn_blobs_tests_a_transactions_chunks_spill_read_back_and_leave_at_its_abort}, + {"change::tests::a_change_of_a_spilled_row_reads_its_spilled_version", + test_txn_change_tests_a_change_of_a_spilled_row_reads_its_spilled_version}, + {"change::tests::a_delete_is_logged_pushed_and_redone_once_and_a_lock_is_never_logged", + test_txn_change_tests_a_delete_is_logged_pushed_and_redone_once_and_a_lock_is_never_logged}, + {"change::tests::a_keys_stripe_is_its_storage_and_bytes_or_its_storage_alone", + test_txn_change_tests_a_keys_stripe_is_its_storage_and_bytes_or_its_storage_alone}, + {"change::tests::an_aborted_delete_and_lock_go_at_the_take_back", + test_txn_change_tests_an_aborted_delete_and_lock_go_at_the_take_back}, {"cport::txn::free_space::tests::an_entry_past_the_bitmap_panics_at_encode", test_txn_free_space_tests_an_entry_past_the_bitmap_panics_at_encode, true, "range end index 250 out of range for slice of length 147"}, @@ -76,12 +102,102 @@ static const CportTest CPORT_TESTS[] = { test_txn_visible_tests_a_running_writers_version_leaves_its_ran_answer_as_it_was}, {"counted::tests::a_keyless_value_is_its_count_and_its_tuple", test_txn_counted_tests_a_keyless_value_is_its_count_and_its_tuple}, + {"entry::tests::a_counted_unique_entry_writes_nothing_for_its_own_copy_and_answers_another", + test_txn_entry_tests_a_counted_unique_entry_writes_nothing_for_its_own_copy_and_answers_another}, + {"entry::tests::a_partial_unique_check_answers_a_running_write_without_waiting", + test_txn_entry_tests_a_partial_unique_check_answers_a_running_write_without_waiting}, + {"entry::tests::a_unique_entry_takes_a_free_key_and_answers_an_existing_entrys_value", + test_txn_entry_tests_a_unique_entry_takes_a_free_key_and_answers_an_existing_entrys_value}, + {"entry::tests::an_entry_and_its_record_change_together", + test_txn_entry_tests_an_entry_and_its_record_change_together}, + {"entry::tests::an_index_base_entry_reads_back_and_refuses_a_keyed_stamp_and_an_uneven_count", + test_txn_entry_tests_an_index_base_entry_reads_back_and_refuses_a_keyed_stamp_and_an_uneven_count}, + {"entry::tests::an_index_entrys_changes_are_logged_pushed_and_redone_once", + test_txn_entry_tests_an_index_entrys_changes_are_logged_pushed_and_redone_once}, + {"entry::tests::an_index_storages_large_keys_spill_at_the_threshold", + test_txn_entry_tests_an_index_storages_large_keys_spill_at_the_threshold}, {"free_space::tests::a_free_space_entry_reads_back_and_refuses_every_other_form", test_txn_free_space_tests_a_free_space_entry_reads_back_and_refuses_every_other_form}, {"free_space::tests::a_seek_finds_the_lowest_block_with_room_through_the_summaries", test_txn_free_space_tests_a_seek_finds_the_lowest_block_with_room_through_the_summaries}, {"free_space::tests::an_insert_takes_the_lowest_free_offset_and_a_left_fresh_page_frees_its_counter_offsets", test_txn_free_space_tests_an_insert_takes_the_lowest_free_offset_and_a_left_fresh_page_frees_its_counter_offsets}, + {"identity::tests::a_columns_id_holds_its_types_category", + test_txn_identity_tests_a_columns_id_holds_its_types_category}, + {"identity::tests::a_pairs_keys_find_it_from_either_side", + test_txn_identity_tests_a_pairs_keys_find_it_from_either_side}, + {"identity::tests::a_saves_drop_takes_its_folded_pairs_out", + test_txn_identity_tests_a_saves_drop_takes_its_folded_pairs_out}, + {"identity::tests::a_saves_fold_keeps_its_committed_pairs_and_leaves_running_ones", + test_txn_identity_tests_a_saves_fold_keeps_its_committed_pairs_and_leaves_running_ones}, + {"identity::tests::a_tables_pairs_read_back_by_its_oid_and_its_id", + test_txn_identity_tests_a_tables_pairs_read_back_by_its_oid_and_its_id}, + {"identity::tests::a_taken_id_moves_on_and_a_removed_pair_frees_it", + test_txn_identity_tests_a_taken_id_moves_on_and_a_removed_pair_frees_it}, + {"identity::tests::an_aborted_transactions_pairs_go_at_the_take_back", + test_txn_identity_tests_an_aborted_transactions_pairs_go_at_the_take_back}, + {"identity::tests::an_identity_change_reads_back_as_written", + test_txn_identity_tests_an_identity_change_reads_back_as_written}, + {"identity::tests::an_objects_id_derives_from_its_kind_parent_and_name", + test_txn_identity_tests_an_objects_id_derives_from_its_kind_parent_and_name}, + {"identity::tests::redone_identity_changes_give_the_same_pairs_once", + test_txn_identity_tests_redone_identity_changes_give_the_same_pairs_once}, + {"insert::tests::a_checkpoint_before_or_after_an_update_leaves_its_kept_counts_once", + test_txn_insert_tests_a_checkpoint_before_or_after_an_update_leaves_its_kept_counts_once}, + {"insert::tests::a_checkpoint_between_an_updates_count_adds_and_its_row_leaves_the_counts_once", + test_txn_insert_tests_a_checkpoint_between_an_updates_count_adds_and_its_row_leaves_the_counts_once}, + {"insert::tests::a_checkpoints_runs_and_every_record_after_redo_as_the_primary", + test_txn_insert_tests_a_checkpoints_runs_and_every_record_after_redo_as_the_primary}, + {"insert::tests::a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both", + test_txn_insert_tests_a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both}, + {"insert::tests::a_frozen_inserts_row_change_replays_with_no_status_room", + test_txn_insert_tests_a_frozen_inserts_row_change_replays_with_no_status_room}, + {"insert::tests::a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter", + test_txn_insert_tests_a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter}, + {"insert::tests::a_kept_values_count_add_is_redone_once", + test_txn_insert_tests_a_kept_values_count_add_is_redone_once}, + {"insert::tests::a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids", + test_txn_insert_tests_a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids}, + {"insert::tests::a_load_moves_the_counter_past_every_loaded_tid_and_left_page", + test_txn_insert_tests_a_load_moves_the_counter_past_every_loaded_tid_and_left_page}, + {"insert::tests::a_resolve_whose_first_target_is_refused_ends_in_bounded_turns", + test_txn_insert_tests_a_resolve_whose_first_target_is_refused_ends_in_bounded_turns}, + {"insert::tests::a_resolve_whose_stripe_moves_three_times_holds_its_writers_back", + test_txn_insert_tests_a_resolve_whose_stripe_moves_three_times_holds_its_writers_back}, + {"insert::tests::a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room", + test_txn_insert_tests_a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room}, + {"insert::tests::a_row_whose_value_references_sit_in_an_overflow_chunk_redoes_its_map_entries", + test_txn_insert_tests_a_row_whose_value_references_sit_in_an_overflow_chunk_redoes_its_map_entries}, + {"insert::tests::a_speculative_insert_holds_its_token_until_confirmed", + test_txn_insert_tests_a_speculative_insert_holds_its_token_until_confirmed}, + {"insert::tests::a_take_redone_again_changes_nothing", + test_txn_insert_tests_a_take_redone_again_changes_nothing}, + {"insert::tests::a_window_between_a_sweeps_rows_and_its_cut_leaves_its_take_without_its_row", + test_txn_insert_tests_a_window_between_a_sweeps_rows_and_its_cut_leaves_its_take_without_its_row}, + {"insert::tests::an_aborted_inserts_value_map_versions_go_at_its_take_back_and_its_free_space_take_stays", + test_txn_insert_tests_an_aborted_inserts_value_map_versions_go_at_its_take_back_and_its_free_space_take_stays}, + {"insert::tests::an_aborted_updates_new_row_and_end_go_at_the_take_back", + test_txn_insert_tests_an_aborted_updates_new_row_and_end_go_at_the_take_back}, + {"insert::tests::an_insert_logical_decoding_reads_logs_heaps_record_of_its_command", + test_txn_insert_tests_an_insert_logical_decoding_reads_logs_heaps_record_of_its_command}, + {"insert::tests::an_insert_maps_its_out_of_line_values_in_its_window", + test_txn_insert_tests_an_insert_maps_its_out_of_line_values_in_its_window}, + {"insert::tests::an_insert_that_fails_after_its_chunks_maps_no_value", + test_txn_insert_tests_an_insert_that_fails_after_its_chunks_maps_no_value}, + {"insert::tests::an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary", + test_txn_insert_tests_an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary}, + {"insert::tests::an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary", + test_txn_insert_tests_an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary}, + {"insert::tests::an_updates_combo_command_shows_on_the_row_it_ends", + test_txn_insert_tests_an_updates_combo_command_shows_on_the_row_it_ends}, + {"insert::tests::inserts_take_heaps_tuple_ids_by_offsets_and_by_bytes", + test_txn_insert_tests_inserts_take_heaps_tuple_ids_by_offsets_and_by_bytes}, + {"insert::tests::redo_from_any_point_gives_a_row_with_all_its_map_entries_or_none", + test_txn_insert_tests_redo_from_any_point_gives_a_row_with_all_its_map_entries_or_none}, + {"insert::tests::redo_leaves_the_free_space_map_and_the_counter_as_the_primarys", + test_txn_insert_tests_redo_leaves_the_free_space_map_and_the_counter_as_the_primarys}, + {"insert::tests::windows_after_a_sweeps_cut_are_in_its_runs_with_neither_their_rows_nor_their_takes", + test_txn_insert_tests_windows_after_a_sweeps_cut_are_in_its_runs_with_neither_their_rows_nor_their_takes}, {"memtable::branch::tests::a_chunk_is_held_by_its_records_lsn", test_txn_memtable_branch_tests_a_chunk_is_held_by_its_records_lsn}, {"memtable::branch::tests::a_chunk_reads_back_by_its_address_until_its_repositorys_drop", @@ -114,6 +230,8 @@ static const CportTest CPORT_TESTS[] = { test_txn_memtable_map_tests_a_failure_after_the_chunks_and_before_the_window_leaves_the_chunk_unreferenced}, {"memtable::map::tests::a_map_only_prepare_takes_its_entry_and_makes_no_rows_memtable", test_txn_memtable_map_tests_a_map_only_prepare_takes_its_entry_and_makes_no_rows_memtable}, + {"memtable::map::tests::a_take_whose_read_moved_answers_moved_and_pushes_nothing", + test_txn_memtable_map_tests_a_take_whose_read_moved_answers_moved_and_pushes_nothing}, {"memtable::map::tests::a_maps_seek_goes_on_past_a_ranges_end", test_txn_memtable_map_tests_a_maps_seek_goes_on_past_a_ranges_end}, {"memtable::map::tests::a_seek_reads_on_after_a_writer_follows_its_read_of_an_empty_map", @@ -192,6 +310,10 @@ static const CportTest CPORT_TESTS[] = { test_txn_memtable_table_tests_a_reclaim_keeps_a_state_whose_maps_are_linked_and_marks_it_due}, {"memtable::table::tests::a_reclaim_keeps_a_state_with_a_memtable", test_txn_memtable_table_tests_a_reclaim_keeps_a_state_with_a_memtable}, + {"memtable::table::tests::a_refused_try_leaves_no_memtable_range_or_page_behind", + test_txn_memtable_table_tests_a_refused_try_leaves_no_memtable_range_or_page_behind}, + {"memtable::table::tests::a_refused_try_of_two_trees_leaves_neither_memtable_behind", + test_txn_memtable_table_tests_a_refused_try_of_two_trees_leaves_neither_memtable_behind}, {"memtable::table::tests::a_removal_closes_an_emptied_range_and_an_emptied_memtable", test_txn_memtable_table_tests_a_removal_closes_an_emptied_range_and_an_emptied_memtable}, {"memtable::table::tests::a_removal_counts_its_records_and_its_keys_bytes_dead", @@ -264,24 +386,74 @@ static const CportTest CPORT_TESTS[] = { test_txn_memtable_table_tests_two_first_writers_make_one_memtable}, {"memtable::table::tests::two_writers_that_find_a_closed_memtable_clear_it_once", test_txn_memtable_table_tests_two_writers_that_find_a_closed_memtable_clear_it_once}, + {"memtable::tuples::tests::a_compaction_failed_at_any_step_leaves_every_tid_found", + test_txn_memtable_tuples_tests_a_compaction_failed_at_any_step_leaves_every_tid_found}, + {"memtable::tuples::tests::a_compaction_with_a_reservation_outstanding_moves_no_byte_page", + test_txn_memtable_tuples_tests_a_compaction_with_a_reservation_outstanding_moves_no_byte_page}, {"memtable::tuples::tests::a_reader_held_across_a_growth_reads_from_the_next_table", test_txn_memtable_tuples_tests_a_reader_held_across_a_growth_reads_from_the_next_table}, {"memtable::tuples::tests::a_reservation_no_finish_stored_is_given_back", test_txn_memtable_tuples_tests_a_reservation_no_finish_stored_is_given_back}, + {"memtable::tuples::tests::a_table_of_mostly_dropped_tids_is_compacted_at_its_load_factor", + test_txn_memtable_tuples_tests_a_table_of_mostly_dropped_tids_is_compacted_at_its_load_factor}, {"memtable::tuples::tests::a_tables_first_pages_are_the_smallest_that_hold_them", test_txn_memtable_tuples_tests_a_tables_first_pages_are_the_smallest_that_hold_them}, {"memtable::tuples::tests::a_tables_tids_are_found_after_inserts_and_a_growth", test_txn_memtable_tuples_tests_a_tables_tids_are_found_after_inserts_and_a_growth}, {"memtable::tuples::tests::a_tid_used_again_names_its_newest_key", test_txn_memtable_tuples_tests_a_tid_used_again_names_its_newest_key}, + {"memtable::tuples::tests::a_writer_waits_on_a_compactions_exclusive_lock", + test_txn_memtable_tuples_tests_a_writer_waits_on_a_compactions_exclusive_lock}, {"memtable::tuples::tests::two_prepares_at_the_bound_each_get_a_slot_through_one_growth", test_txn_memtable_tuples_tests_two_prepares_at_the_bound_each_get_a_slot_through_one_growth}, {"memtable::tuples::tests::two_tids_of_one_first_slot_are_each_found", test_txn_memtable_tuples_tests_two_tids_of_one_first_slot_are_each_found}, + {"memtable::version::tests::a_hint_and_a_token_clear_racing_both_survive", + test_txn_memtable_version_tests_a_hint_and_a_token_clear_racing_both_survive}, + {"memtable::version::tests::a_keys_versions_read_newest_first_with_their_deleters", + test_txn_memtable_version_tests_a_keys_versions_read_newest_first_with_their_deleters}, + {"memtable::version::tests::a_link_holds_its_offset_and_older_handle_at_their_limits", + test_txn_memtable_version_tests_a_link_holds_its_offset_and_older_handle_at_their_limits}, + {"memtable::version::tests::a_shown_combo_is_the_rows_raw_command_and_no_token", + test_txn_memtable_version_tests_a_shown_combo_is_the_rows_raw_command_and_no_token}, + {"memtable::version::tests::a_stored_record_is_refused_unless_encode_writes_it", + test_txn_memtable_version_tests_a_stored_record_is_refused_unless_encode_writes_it}, + {"memtable::version::tests::a_version_on_a_fresh_page_is_named_by_its_index_in_the_list", + test_txn_memtable_version_tests_a_version_on_a_fresh_page_is_named_by_its_index_in_the_list}, + {"memtable::version::tests::every_ops_record_reads_back_as_written", + test_txn_memtable_version_tests_every_ops_record_reads_back_as_written}, + {"outcome::tests::a_writers_outcome_answers_frozen_with_no_door_and_caches_an_ended_writers", + test_txn_outcome_tests_a_writers_outcome_answers_frozen_with_no_door_and_caches_an_ended_writers}, + {"record::tests::a_built_bases_node_reads_back_with_its_stamps", + test_txn_record_tests_a_built_bases_node_reads_back_with_its_stamps}, + {"record::tests::a_built_nodes_key_names_its_base_and_its_fields_come_together", + test_txn_record_tests_a_built_nodes_key_names_its_base_and_its_fields_come_together}, + {"record::tests::a_chunk_reads_back_and_refuses_every_other_form", + test_txn_record_tests_a_chunk_reads_back_and_refuses_every_other_form}, + {"record::tests::a_folds_record_reads_back_and_refuses_every_other_form", + test_txn_record_tests_a_folds_record_reads_back_and_refuses_every_other_form}, + {"record::tests::a_free_reads_back_and_refuses_runs_out_of_bounds_or_order", + test_txn_record_tests_a_free_reads_back_and_refuses_runs_out_of_bounds_or_order}, + {"record::tests::a_frees_taken_down_values_read_back_and_refuse_every_other_form", + test_txn_record_tests_a_frees_taken_down_values_read_back_and_refuse_every_other_form}, + {"record::tests::a_moves_carried_free_space_reads_back_and_refuses_every_other_form", + test_txn_record_tests_a_moves_carried_free_space_reads_back_and_refuses_every_other_form}, + {"record::tests::a_page_leave_reads_back_and_refuses_every_other_form", + test_txn_record_tests_a_page_leave_reads_back_and_refuses_every_other_form}, + {"record::tests::a_row_change_reads_back_and_refuses_every_other_form", + test_txn_record_tests_a_row_change_reads_back_and_refuses_every_other_form}, + {"record::tests::an_index_entrys_change_reads_back_and_refuses_every_field_its_op_doesnt_carry", + test_txn_record_tests_an_index_entrys_change_reads_back_and_refuses_every_field_its_op_doesnt_carry}, + {"record::tests::an_updates_kept_values_read_back_in_their_values_order", + test_txn_record_tests_an_updates_kept_values_read_back_in_their_values_order}, + {"record::tests::stamps_on_a_blob_or_of_another_count_are_refused", + test_txn_record_tests_stamps_on_a_blob_or_of_another_count_are_refused}, {"runs::tests::a_checkpoint_names_a_running_spill_and_a_start_links_it_back", test_txn_runs_tests_a_checkpoint_names_a_running_spill_and_a_start_links_it_back}, {"runs::tests::a_checkpoint_saves_its_runs_and_removes_the_unnamed", test_txn_runs_tests_a_checkpoint_saves_its_runs_and_removes_the_unnamed}, + {"runs::tests::a_checkpoints_identity_run_loads_its_pairs_back", + test_txn_runs_tests_a_checkpoints_identity_run_loads_its_pairs_back}, {"runs::tests::a_checkpoints_runs_hold_each_trees_versions", test_txn_runs_tests_a_checkpoints_runs_hold_each_trees_versions}, {"runs::tests::a_checkpoints_runs_load_back_into_the_delta", @@ -290,8 +462,16 @@ static const CportTest CPORT_TESTS[] = { test_txn_runs_tests_a_checkpoints_runs_survive_a_power_loss}, {"runs::tests::a_checkpoints_status_run_gives_the_loaded_writers_their_outcomes", test_txn_runs_tests_a_checkpoints_status_run_gives_the_loaded_writers_their_outcomes}, + {"runs::tests::a_deferred_spill_named_again_waits_for_two_new_sightings", + test_txn_runs_tests_a_deferred_spill_named_again_waits_for_two_new_sightings}, + {"runs::tests::a_helpers_part_linked_after_its_transactions_abort_leaves_with_its_files", + test_txn_runs_tests_a_helpers_part_linked_after_its_transactions_abort_leaves_with_its_files}, {"runs::tests::a_large_status_run_keeps_every_entry_through_its_load", test_txn_runs_tests_a_large_status_run_keeps_every_entry_through_its_load}, + {"runs::tests::a_pair_the_save_folds_is_left_out_of_the_identity_run", + test_txn_runs_tests_a_pair_the_save_folds_is_left_out_of_the_identity_run}, + {"runs::tests::a_reader_that_listed_a_merges_source_reads_it_across_two_checkpoints", + test_txn_runs_tests_a_reader_that_listed_a_merges_source_reads_it_across_two_checkpoints}, {"runs::tests::a_recoverys_cleanup_drops_an_unlogged_storages_loaded_rows", test_txn_runs_tests_a_recoverys_cleanup_drops_an_unlogged_storages_loaded_rows}, {"runs::tests::a_recoverys_cleanup_resets_an_unlogged_storages_saved_base", @@ -322,6 +502,8 @@ static const CportTest CPORT_TESTS[] = { test_txn_runs_tests_a_start_skips_a_repository_whose_run_is_damaged_and_loads_the_others}, {"runs::tests::a_start_warns_of_a_manifest_it_cant_read_and_loads_the_others_runs", test_txn_runs_tests_a_start_warns_of_a_manifest_it_cant_read_and_loads_the_others_runs}, + {"runs::tests::a_starts_loaded_identity_pairs_count_at_their_writers_entry", + test_txn_runs_tests_a_starts_loaded_identity_pairs_count_at_their_writers_entry}, {"runs::tests::a_status_run_reads_back_its_outcomes_and_refuses_what_isnt_one", test_txn_runs_tests_a_status_run_reads_back_its_outcomes_and_refuses_what_isnt_one}, {"runs::tests::a_takes_ender_loads_back_from_its_free_space_run", @@ -330,10 +512,24 @@ static const CportTest CPORT_TESTS[] = { test_txn_runs_tests_an_entries_run_reads_its_entries_and_refuses_heap_bytes_tuple_ids_and_other_ops}, {"runs::tests::an_entries_runs_header_holds_its_table_and_index", test_txn_runs_tests_an_entries_runs_header_holds_its_table_and_index}, + {"runs::tests::an_identity_run_reads_its_pairs_and_refuses_heap_bytes_tuple_ids_and_other_ops", + test_txn_runs_tests_an_identity_run_reads_its_pairs_and_refuses_heap_bytes_tuple_ids_and_other_ops}, + {"runs::tests::an_identity_spill_reads_its_pairs_and_refuses_another_writer_tuple_ids_and_heap_bytes", + test_txn_runs_tests_an_identity_spill_reads_its_pairs_and_refuses_another_writer_tuple_ids_and_heap_bytes}, + {"runs::tests::an_index_storages_entries_spill_and_load_back_as_entries", + test_txn_runs_tests_an_index_storages_entries_spill_and_load_back_as_entries}, {"runs::tests::an_index_storages_loaded_entries_stay_an_entries_run_through_a_second_checkpoint", test_txn_runs_tests_an_index_storages_loaded_entries_stay_an_entries_run_through_a_second_checkpoint}, + {"spill::tests::a_carry_that_fails_at_any_step_lists_nothing_and_leaves_every_count", + test_txn_spill_tests_a_carry_that_fails_at_any_step_lists_nothing_and_leaves_every_count}, + {"spill::tests::a_drop_counts_a_crashed_transactions_files_in_either_order", + test_txn_spill_tests_a_drop_counts_a_crashed_transactions_files_in_either_order}, {"spill::tests::a_filter_hits_its_keys_and_few_others", test_txn_spill_tests_a_filter_hits_its_keys_and_few_others}, + {"spill::tests::a_free_space_spill_a_crash_ended_is_marked_with_the_next_checkpoints_redo_point", + test_txn_spill_tests_a_free_space_spill_a_crash_ended_is_marked_with_the_next_checkpoints_redo_point}, + {"spill::tests::a_free_space_spills_merge_leaves_the_rows_tuple_ids_alone", + test_txn_spill_tests_a_free_space_spills_merge_leaves_the_rows_tuple_ids_alone}, {"spill::tests::a_large_filter_takes_several_pages", test_txn_spill_tests_a_large_filter_takes_several_pages}, {"spill::tests::a_lookup_reads_an_evicted_filter_page_again", @@ -348,12 +544,16 @@ static const CportTest CPORT_TESTS[] = { test_txn_spill_tests_a_one_range_tree_spills_at_a_full_budget}, {"spill::tests::a_pending_spill_is_found_by_its_storages_readers_until_it_settles", test_txn_spill_tests_a_pending_spill_is_found_by_its_storages_readers_until_it_settles}, + {"spill::tests::a_prepared_transactions_spills_are_never_taken_out", + test_txn_spill_tests_a_prepared_transactions_spills_are_never_taken_out}, {"spill::tests::a_process_with_nothing_of_its_own_spills_a_paused_transactions_versions", test_txn_spill_tests_a_process_with_nothing_of_its_own_spills_a_paused_transactions_versions}, {"spill::tests::a_reader_across_a_merges_unchain_reads_each_version_once", test_txn_spill_tests_a_reader_across_a_merges_unchain_reads_each_version_once}, {"spill::tests::a_reader_paused_in_a_manifest_read_finds_each_version_once", test_txn_spill_tests_a_reader_paused_in_a_manifest_read_finds_each_version_once}, + {"spill::tests::a_residual_whose_listing_fails_lists_nothing_and_gives_its_count_back", + test_txn_spill_tests_a_residual_whose_listing_fails_lists_nothing_and_gives_its_count_back}, {"spill::tests::a_scan_merges_spilled_keys_across_chunks_in_order", test_txn_spill_tests_a_scan_merges_spilled_keys_across_chunks_in_order}, {"spill::tests::a_second_spill_merges_with_the_first_keeping_its_versions_order", @@ -364,6 +564,8 @@ static const CportTest CPORT_TESTS[] = { test_txn_spill_tests_a_spill_of_many_keys_writes_one_file_whose_filter_takes_pages}, {"spill::tests::a_spill_takes_only_the_keys_its_transaction_alone_holds", test_txn_spill_tests_a_spill_takes_only_the_keys_its_transaction_alone_holds}, + {"spill::tests::a_spilled_free_space_maps_entries_are_read_below_the_memtable", + test_txn_spill_tests_a_spilled_free_space_maps_entries_are_read_below_the_memtable}, {"spill::tests::a_spilled_value_maps_entries_are_read_below_the_memtable", test_txn_spill_tests_a_spilled_value_maps_entries_are_read_below_the_memtable}, {"spill::tests::a_spills_tuple_ids_still_find_their_rows_keys", @@ -374,12 +576,18 @@ static const CportTest CPORT_TESTS[] = { test_txn_spill_tests_a_transaction_spilling_more_storages_than_the_budget_holds_pages_completes}, {"spill::tests::an_abort_takes_its_spills_out_of_the_chain", test_txn_spill_tests_an_abort_takes_its_spills_out_of_the_chain}, + {"spill::tests::an_aborted_free_space_map_spill_stays_listed_until_a_fold", + test_txn_spill_tests_an_aborted_free_space_map_spill_stays_listed_until_a_fold}, {"spill::tests::an_owner_spills_nothing_while_a_helper_holds_its_transactions_turn", test_txn_spill_tests_an_owner_spills_nothing_while_a_helper_holds_its_transactions_turn}, {"spill::tests::more_transactions_than_any_fixed_count_spill_one_storage_at_a_full_budget", test_txn_spill_tests_more_transactions_than_any_fixed_count_spill_one_storage_at_a_full_budget}, {"spill::tests::redo_over_budget_spills_the_largest_replayed_transaction", test_txn_spill_tests_redo_over_budget_spills_the_largest_replayed_transaction}, + {"spill::tests::the_committed_arm_spills_a_keys_oldest_committed_versions_above_its_spills", + test_txn_spill_tests_the_committed_arm_spills_a_keys_oldest_committed_versions_above_its_spills}, + {"spill::tests::the_committed_arms_spill_reads_back_in_order_after_a_restart", + test_txn_spill_tests_the_committed_arms_spill_reads_back_in_order_after_a_restart}, {"spill::tests::three_writers_of_small_trees_past_a_third_of_the_budget_each_spill_and_finish", test_txn_spill_tests_three_writers_of_small_trees_past_a_third_of_the_budget_each_spill_and_finish}, {"spill::tests::three_writers_waiting_on_committed_versions_finish_once_a_fold_installs", @@ -392,8 +600,12 @@ static const CportTest CPORT_TESTS[] = { test_txn_state_tests_a_backend_records_its_key_in_a_state_and_redo_takes_none}, {"state::tests::a_linked_free_space_map_or_tuple_id_table_keeps_the_counter_from_falling_back", test_txn_state_tests_a_linked_free_space_map_or_tuple_id_table_keeps_the_counter_from_falling_back}, + {"state::tests::a_make_while_running_writers_hold_every_unsaved_state_gets_the_pools_error", + test_txn_state_tests_a_make_while_running_writers_hold_every_unsaved_state_gets_the_pools_error}, {"state::tests::a_maker_terminated_before_its_fill_leaves_the_fill_to_the_next_finder", test_txn_state_tests_a_maker_terminated_before_its_fill_leaves_the_fill_to_the_next_finder}, + {"state::tests::a_rechain_keeps_a_key_recorded_while_it_logs", + test_txn_state_tests_a_rechain_keeps_a_key_recorded_while_it_logs}, {"state::tests::a_reclaim_keeps_a_state_whose_manifest_lists_a_spill", test_txn_state_tests_a_reclaim_keeps_a_state_whose_manifest_lists_a_spill}, {"state::tests::a_save_gathers_a_linked_states_counter_and_lets_an_idle_ones_fall_back", @@ -406,6 +618,8 @@ static const CportTest CPORT_TESTS[] = { test_txn_state_tests_a_state_opened_while_a_reclaim_looks_at_it_is_kept}, {"state::tests::a_state_saved_reclaimed_and_made_again_reads_the_same", test_txn_state_tests_a_state_saved_reclaimed_and_made_again_reads_the_same}, + {"state::tests::a_state_with_spills_keeps_the_counter_from_falling_back", + test_txn_state_tests_a_state_with_spills_keeps_the_counter_from_falling_back}, {"state::tests::a_write_between_a_gathering_and_its_saved_leaves_the_state_unsaved", test_txn_state_tests_a_write_between_a_gathering_and_its_saved_leaves_the_state_unsaved}, {"state::tests::an_install_keeps_a_key_recorded_after_its_fold_read_the_base", @@ -422,6 +636,136 @@ static const CportTest CPORT_TESTS[] = { test_txn_state_tests_the_pool_holds_a_state_for_each_relation_lock_and_reclaims_idle_ones}, {"state::tests::two_backends_that_open_one_storage_make_one_state", test_txn_state_tests_two_backends_that_open_one_storage_make_one_state}, + {"status::tests::a_commit_registered_in_its_slot_reads_back_and_its_move_settles_the_entry", + test_txn_status_tests_a_commit_registered_in_its_slot_reads_back_and_its_move_settles_the_entry}, + {"status::tests::a_folds_point_lies_below_a_commit_still_in_the_proc_array", + test_txn_status_tests_a_folds_point_lies_below_a_commit_still_in_the_proc_array}, + {"status::tests::a_growth_holds_what_it_keeps_in_the_least_room", + test_txn_status_tests_a_growth_holds_what_it_keeps_in_the_least_room}, + {"status::tests::a_growth_while_the_proc_array_cant_answer_keeps_an_ended_entry", + test_txn_status_tests_a_growth_while_the_proc_array_cant_answer_keeps_an_ended_entry}, + {"status::tests::a_lookup_probes_every_place_and_neighbours_start_apart", + test_txn_status_tests_a_lookup_probes_every_place_and_neighbours_start_apart}, + {"status::tests::a_prepared_commit_points_its_entry_at_the_committer_and_registers", + test_txn_status_tests_a_prepared_commit_points_its_entry_at_the_committer_and_registers}, + {"status::tests::a_replayed_commit_stores_its_end_in_the_entry_its_replay_took", + test_txn_status_tests_a_replayed_commit_stores_its_end_in_the_entry_its_replay_took}, + {"status::tests::a_slot_stays_its_entrys_across_a_growth_and_a_removed_ones_is_taken_again", + test_txn_status_tests_a_slot_stays_its_entrys_across_a_growth_and_a_removed_ones_is_taken_again}, + {"status::tests::a_table_is_crowded_at_three_quarters", + test_txn_status_tests_a_table_is_crowded_at_three_quarters}, + {"status::tests::a_take_of_room_moves_the_slots_pair_first", + test_txn_status_tests_a_take_of_room_moves_the_slots_pair_first}, + {"status::tests::an_outcome_too_large_for_its_word_has_none", + test_txn_status_tests_an_outcome_too_large_for_its_word_has_none}, + {"status::tests::an_outcomes_word_says_the_outcome", + test_txn_status_tests_an_outcomes_word_says_the_outcome}, + {"status::tests::an_xids_word_is_no_empty_places", + test_txn_status_tests_an_xids_word_is_no_empty_places}, + {"status::tests::churn_with_no_live_entries_keeps_one_segment", + test_txn_status_tests_churn_with_no_live_entries_keeps_one_segment}, + {"status::tests::entries_are_taken_found_counted_removed_and_kept_by_a_growth", + test_txn_status_tests_entries_are_taken_found_counted_removed_and_kept_by_a_growth}, + {"vacuum::tests::a_dead_tuple_id_is_claimed_once", + test_txn_vacuum_tests_a_dead_tuple_id_is_claimed_once}, + {"vacuum::tests::a_dead_tuple_id_on_a_full_block_keeps_its_bytes_until_its_free", + test_txn_vacuum_tests_a_dead_tuple_id_on_a_full_block_keeps_its_bytes_until_its_free}, + {"vacuum::tests::a_dropped_rows_tuple_id_keeps_its_enders_xid_for_its_frees_horizon", + test_txn_vacuum_tests_a_dropped_rows_tuple_id_keeps_its_enders_xid_for_its_frees_horizon}, + {"vacuum::tests::a_dropped_rows_tuple_id_waits_for_the_horizon_to_pass_its_ender", + test_txn_vacuum_tests_a_dropped_rows_tuple_id_waits_for_the_horizon_to_pass_its_ender}, + {"vacuum::tests::a_drops_ender_goes_on_its_rows_take_not_a_later_take_of_the_tuple_id", + test_txn_vacuum_tests_a_drops_ender_goes_on_its_rows_take_not_a_later_take_of_the_tuple_id}, + {"vacuum::tests::a_fold_between_a_prunes_reads_leaves_what_a_prune_after_it_leaves", + test_txn_vacuum_tests_a_fold_between_a_prunes_reads_leaves_what_a_prune_after_it_leaves}, + {"vacuum::tests::a_fold_between_the_finds_reads_leaves_its_rows_held", + test_txn_vacuum_tests_a_fold_between_the_finds_reads_leaves_its_rows_held}, + {"vacuum::tests::a_fold_during_a_blocks_spent_count_counts_what_a_count_after_it_counts", + test_txn_vacuum_tests_a_fold_during_a_blocks_spent_count_counts_what_a_count_after_it_counts}, + {"vacuum::tests::a_fold_during_a_find_finds_what_a_find_after_it_finds", + test_txn_vacuum_tests_a_fold_during_a_find_finds_what_a_find_after_it_finds}, + {"vacuum::tests::a_fold_during_a_free_leaves_what_a_free_after_it_leaves", + test_txn_vacuum_tests_a_fold_during_a_free_leaves_what_a_free_after_it_leaves}, + {"vacuum::tests::a_fold_during_a_frees_redo_leaves_what_a_redo_after_it_leaves", + test_txn_vacuum_tests_a_fold_during_a_frees_redo_leaves_what_a_redo_after_it_leaves}, + {"vacuum::tests::a_fold_during_a_frees_redo_leaves_what_the_free_left", + test_txn_vacuum_tests_a_fold_during_a_frees_redo_leaves_what_the_free_left}, + {"vacuum::tests::a_folded_rows_tuple_id_is_not_freed", + test_txn_vacuum_tests_a_folded_rows_tuple_id_is_not_freed}, + {"vacuum::tests::a_free_of_a_block_whose_takes_all_spilled_starts_from_its_newest_spilled_entry", + test_txn_vacuum_tests_a_free_of_a_block_whose_takes_all_spilled_starts_from_its_newest_spilled_entry}, + {"vacuum::tests::a_frees_enders_of_another_form_are_refused_as_corruption", + test_txn_vacuum_tests_a_frees_enders_of_another_form_are_refused_as_corruption}, + {"vacuum::tests::a_frees_take_out_leaves_a_retake_on_its_key", + test_txn_vacuum_tests_a_frees_take_out_leaves_a_retake_on_its_key}, + {"vacuum::tests::a_frees_take_out_takes_out_a_subtransactions_delete", + test_txn_vacuum_tests_a_frees_take_out_takes_out_a_subtransactions_delete}, + {"vacuum::tests::a_frees_take_out_with_no_ender_leaves_any_retake_on_its_key", + test_txn_vacuum_tests_a_frees_take_out_with_no_ender_leaves_any_retake_on_its_key}, + {"vacuum::tests::a_prune_over_an_aborted_unfolded_insert_claims_nothing", + test_txn_vacuum_tests_a_prune_over_an_aborted_unfolded_insert_claims_nothing}, + {"vacuum::tests::a_prunes_take_out_of_an_aborted_writers_own_row_leaves_a_retake", + test_txn_vacuum_tests_a_prunes_take_out_of_an_aborted_writers_own_row_leaves_a_retake}, + {"vacuum::tests::a_refused_markers_redo_keeps_the_version_as_the_primary_does", + test_txn_vacuum_tests_a_refused_markers_redo_keeps_the_version_as_the_primary_does}, + {"vacuum::tests::a_row_its_own_transaction_inserted_and_deleted_waits_for_the_horizon", + test_txn_vacuum_tests_a_row_its_own_transaction_inserted_and_deleted_waits_for_the_horizon}, + {"vacuum::tests::a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone", + test_txn_vacuum_tests_a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone}, + {"vacuum::tests::a_take_leaves_a_version_an_open_snapshot_may_see", + test_txn_vacuum_tests_a_take_leaves_a_version_an_open_snapshot_may_see}, + {"vacuum::tests::a_take_no_row_names_is_freed_only_once_its_taker_has_ended", + test_txn_vacuum_tests_a_take_no_row_names_is_freed_only_once_its_taker_has_ended}, + {"vacuum::tests::a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block", + test_txn_vacuum_tests_a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block}, + {"vacuum::tests::a_take_that_empties_its_entry_is_freed_with_all_the_bytes_the_entry_held", + test_txn_vacuum_tests_a_take_that_empties_its_entry_is_freed_with_all_the_bytes_the_entry_held}, + {"vacuum::tests::a_takes_ender_widens_against_its_writer_across_an_epoch", + test_txn_vacuum_tests_a_takes_ender_widens_against_its_writer_across_an_epoch}, + {"vacuum::tests::an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again", + test_txn_vacuum_tests_an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again}, + {"vacuum::tests::an_aborted_leaves_old_offsets_and_its_new_tuple_id_are_handed_out_again", + test_txn_vacuum_tests_an_aborted_leaves_old_offsets_and_its_new_tuple_id_are_handed_out_again}, + {"vacuum::tests::every_raise_of_a_blocks_room_lowers_its_categorys_hint", + test_txn_vacuum_tests_every_raise_of_a_blocks_room_lowers_its_categorys_hint}, + {"values::tests::a_take_down_takes_one_from_a_count_and_removes_the_last", + test_txn_values_tests_a_take_down_takes_one_from_a_count_and_removes_the_last}, + {"values::tests::a_value_counts_heaps_toast_chunks", + test_txn_values_tests_a_value_counts_heaps_toast_chunks}, + {"values::tests::a_value_id_the_map_holds_is_skipped", + test_txn_values_tests_a_value_id_the_map_holds_is_skipped}, + {"values::tests::a_value_ids_entry_reads_back", + test_txn_values_tests_a_value_ids_entry_reads_back}, + {"values::tests::a_value_map_entry_reads_back_and_refuses_every_other_form", + test_txn_values_tests_a_value_map_entry_reads_back_and_refuses_every_other_form}, + {"visible::tests::a_historic_snapshot_sees_its_transactions_rows_by_their_commands", + test_txn_visible_tests_a_historic_snapshot_sees_its_transactions_rows_by_their_commands}, + {"visible::tests::a_key_read_gives_every_row_version_the_snapshots_rule_sees", + test_txn_visible_tests_a_key_read_gives_every_row_version_the_snapshots_rule_sees}, + {"visible::tests::a_killed_speculative_insert_is_aborted_for_every_reader_at_once", + test_txn_visible_tests_a_killed_speculative_insert_is_aborted_for_every_reader_at_once}, + {"visible::tests::a_row_named_by_its_tuple_id_is_ended_by_the_version_that_names_it", + test_txn_visible_tests_a_row_named_by_its_tuple_id_is_ended_by_the_version_that_names_it}, + {"visible::tests::a_rows_change_is_answered_as_heaps_satisfies_update_answers", + test_txn_visible_tests_a_rows_change_is_answered_as_heaps_satisfies_update_answers}, + {"visible::tests::a_rows_delete_is_seen_only_by_a_snapshot_taken_after_its_commit", + test_txn_visible_tests_a_rows_delete_is_seen_only_by_a_snapshot_taken_after_its_commit}, + {"visible::tests::a_rows_slice_after_a_removal_under_its_head_answers_as_its_whole_chain", + test_txn_visible_tests_a_rows_slice_after_a_removal_under_its_head_answers_as_its_whole_chain}, + {"visible::tests::a_rows_slice_answers_as_its_whole_chain", + test_txn_visible_tests_a_rows_slice_answers_as_its_whole_chain}, + {"visible::tests::a_rows_stamp_shows_the_last_change_above_it_whatever_its_outcome", + test_txn_visible_tests_a_rows_stamp_shows_the_last_change_above_it_whatever_its_outcome}, + {"visible::tests::a_serializable_read_conflicts_out_with_the_writers_heaps_rule_names", + test_txn_visible_tests_a_serializable_read_conflicts_out_with_the_writers_heaps_rule_names}, + {"visible::tests::a_subtransactions_row_stays_invisible_to_a_snapshot_taken_while_its_transaction_ran", + test_txn_visible_tests_a_subtransactions_row_stays_invisible_to_a_snapshot_taken_while_its_transaction_ran}, + {"visible::tests::a_transactions_own_row_is_seen_from_the_command_after_its_write", + test_txn_visible_tests_a_transactions_own_row_is_seen_from_the_command_after_its_write}, + {"visible::tests::a_versions_epoch_counts_the_transforms_written_before_it", + test_txn_visible_tests_a_versions_epoch_counts_the_transforms_written_before_it}, + {"visible::tests::each_snapshot_kind_sees_the_versions_heaps_rule_sees", + test_txn_visible_tests_each_snapshot_kind_sees_the_versions_heaps_rule_sees}, }; PG_FUNCTION_INFO_V1(doltgres_cport_test_txn); diff --git a/src/doltgres/test/modules/doltgres_test/doltgres_test--1.0.sql b/src/doltgres/test/modules/doltgres_test/doltgres_test--1.0.sql index 4c26ee5aeefa8..6bf378a4db871 100644 --- a/src/doltgres/test/modules/doltgres_test/doltgres_test--1.0.sql +++ b/src/doltgres/test/modules/doltgres_test/doltgres_test--1.0.sql @@ -332,3 +332,18 @@ AS 'MODULE_PATHNAME', 'dg_listed_unwritten' LANGUAGE C STRICT; CREATE FUNCTION doltgres_test_step_unwritten() RETURNS bool AS 'MODULE_PATHNAME', 'dg_step_unwritten' LANGUAGE C STRICT; + +CREATE FUNCTION doltgres_test_scan_shape(target regclass) RETURNS text +AS 'MODULE_PATHNAME', 'dg_scan_shape' LANGUAGE C STRICT; + +CREATE FUNCTION doltgres_test_index_move(index regclass, tablespace text) RETURNS void +AS 'MODULE_PATHNAME', 'dg_index_move' LANGUAGE C STRICT; + +CREATE FUNCTION doltgres_test_base_points(target regclass) RETURNS text +AS 'MODULE_PATHNAME', 'dg_base_points' LANGUAGE C STRICT; + +CREATE FUNCTION doltgres_test_identity_versions(kind int4, number oid) RETURNS text +AS 'MODULE_PATHNAME', 'dg_identity_versions' LANGUAGE C STRICT; + +CREATE FUNCTION doltgres_test_save() RETURNS text +AS 'MODULE_PATHNAME', 'dg_save' LANGUAGE C STRICT; diff --git a/src/doltgres/test/modules/doltgres_test/doltgres_test.c b/src/doltgres/test/modules/doltgres_test/doltgres_test.c index 56587bb2f658d..6759ea72ecf08 100644 --- a/src/doltgres/test/modules/doltgres_test/doltgres_test.c +++ b/src/doltgres/test/modules/doltgres_test/doltgres_test.c @@ -44,11 +44,13 @@ #include "catalog/pg_class.h" #include "catalog/pg_constraint.h" #include "catalog/pg_type_d.h" +#include "catalog/storage.h" #include "commands/progress.h" #include "commands/repack.h" #include "commands/tablecmds.h" #include "commands/tablespace.h" #include "commands/vacuum.h" +#include "doltgres/hooks.h" #include "executor/executor.h" #include "executor/spi.h" #include "executor/tuptable.h" @@ -124,7 +126,11 @@ extern void doltgres_test_index_fetch_key(struct IndexFetchTableData *fetch, Rel extern bool doltgres_test_fold(Relation target); extern Datum doltgres_test_base_sizes(Relation target); extern Datum doltgres_test_totals(Relation target); +extern Datum doltgres_test_scan_shape(Relation target); extern Datum doltgres_test_count(Relation target); +extern Datum doltgres_test_base_points(Relation target); +extern Datum doltgres_test_identity_versions(int32 kind, Oid number); +extern Datum doltgres_test_save(void); extern Datum doltgres_test_index_entries(Relation target); extern bool doltgres_test_held(Relation target, uint64 tid); extern Datum doltgres_test_index_base(Relation target); @@ -252,6 +258,11 @@ PG_FUNCTION_INFO_V1(dg_totals); PG_FUNCTION_INFO_V1(dg_count); PG_FUNCTION_INFO_V1(dg_drop_constraint); PG_FUNCTION_INFO_V1(dg_unchain); +PG_FUNCTION_INFO_V1(dg_scan_shape); +PG_FUNCTION_INFO_V1(dg_index_move); +PG_FUNCTION_INFO_V1(dg_base_points); +PG_FUNCTION_INFO_V1(dg_identity_versions); +PG_FUNCTION_INFO_V1(dg_save); /* dg_store writes the shared area's root record under the fork's lock. */ Datum @@ -2709,3 +2720,77 @@ dg_unchain(PG_FUNCTION_ARGS) ProcessUtility_hook = unchained_utility; PG_RETURN_VOID(); } + +/* dg_scan_shape returns a fork table's shape as its scans' costs read it, as doltgres_test_scan_shape reads it. */ +Datum +dg_scan_shape(PG_FUNCTION_ARGS) +{ + Relation rel = table_open(PG_GETARG_OID(0), AccessShareLock); + Datum shape = doltgres_test_scan_shape(rel); + + table_close(rel, AccessShareLock); + PG_RETURN_DATUM(shape); +} + +/* + * dg_index_move moves an index into a tablespace as ALTER INDEX SET TABLESPACE's index_copy_data does: a new + * relfilenode with its main fork's pages copied, the fork's move of its storage, the old file dropped at commit, and + * its pg_class row. + */ +Datum +dg_index_move(PG_FUNCTION_ARGS) +{ + Relation rel = index_open(PG_GETARG_OID(0), AccessExclusiveLock); + Oid space = get_tablespace_oid(text_to_cstring(PG_GETARG_TEXT_PP(1)), false); + RelFileNumber number = GetNewRelFileNumber(space, NULL, rel->rd_rel->relpersistence); + RelFileLocator locator = rel->rd_locator; + SMgrRelation dstrel; + + locator.relNumber = number; + locator.spcOid = space; + dstrel = RelationCreateStorage(locator, rel->rd_rel->relpersistence, true); + RelationCopyStorage(RelationGetSmgr(rel), dstrel, MAIN_FORKNUM, rel->rd_rel->relpersistence); + doltgres_index_copy_data(rel, &locator); + RelationDropStorage(rel); + smgrclose(dstrel); + SetRelationTableSpace(rel, space, number); + RelationAssumeNewRelfilelocator(rel); + index_close(rel, NoLock); + CommandCounterIncrement(); + PG_RETURN_VOID(); +} + +/* + * dg_base_points returns a fork table's newest base's root, fold point, and read point, as doltgres_test_base_points + * reads them. + */ +Datum +dg_base_points(PG_FUNCTION_ARGS) +{ + Relation rel = table_open(PG_GETARG_OID(0), AccessShareLock); + Datum points = doltgres_test_base_points(rel); + + table_close(rel, AccessShareLock); + PG_RETURN_DATUM(points); +} + +/* + * dg_identity_versions returns how many versions the delta and its identity + * spills hold of an object's identity key by its number, as + * doltgres_test_identity_versions counts them. + */ +Datum +dg_identity_versions(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(doltgres_test_identity_versions(PG_GETARG_INT32(0), PG_GETARG_OID(1))); +} + +/* + * dg_save saves the database's repository outside a checkpoint, as + * doltgres_test_save does. + */ +Datum +dg_save(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(doltgres_test_save()); +} diff --git a/src/doltgres/test/t/006_wal.pl b/src/doltgres/test/t/006_wal.pl index 1345a3dc07acc..f8b21eed755fd 100644 --- a/src/doltgres/test/t/006_wal.pl +++ b/src/doltgres/test/t/006_wal.pl @@ -129,14 +129,14 @@ "\\set VERBOSITY verbose\nSELECT doltgres_test_wal_unknown(0)"); like( $stderr, - qr/ERROR: XX000: versioned WAL record of kind 14 holds no data/, + qr/ERROR: XX000: versioned WAL record of kind \d+ holds no data/, 'a record without data is refused'); $node->safe_psql('postgres', 'SELECT doltgres_test_wal_unknown(1)'); $node->stop('immediate'); ok(!$node->start(fail_ok => 1), 'a record of an unknown kind stops recovery'); like( slurp_file($node->logfile), - qr/PANIC: versioned_redo: unknown op code 224/, + qr/PANIC: versioned_redo: unknown op code \d+/, 'it names the unknown op code'); # Redo, outside any transaction, is refused a checkpoint delay, which a diff --git a/src/doltgres/test/t/017_pages.pl b/src/doltgres/test/t/017_pages.pl index 94019f01ed4a6..8215ee75e3cfd 100644 --- a/src/doltgres/test/t/017_pages.pl +++ b/src/doltgres/test/t/017_pages.pl @@ -273,15 +273,16 @@ sub restamped # The region is listed at its blocks, the setting's sixteen and one more for # the copies compactions take past the budget (bugs rows 110 and 116), and the -# directory at its header, ten cache lines, a cache line for each 1 KiB page +# directory at its header, eleven cache lines, a cache line for each 1 KiB page # the region's blocks can hold, that block's included, three words for each -# block, and a word of the free hint for each 64 slots below the block past -# the budget's (bugs row 150). +# block, a word of the free hint for each 64 slots below the block past the +# budget's (bugs row 150), and a slot's four-byte link in the retired list for +# each slot (bugs row 309). is( $node->safe_psql( 'postgres', "SELECT name, size FROM pg_shmem_allocations WHERE name LIKE 'versioned delta%' ORDER BY name" ), - "versioned delta|1114112\nversioned delta directory|70808", + "versioned delta|1114112\nversioned delta directory|75224", "the delta's region and its directory are listed with their sizes"); # On the fresh server the region gives sixteen pages of a block at once, the @@ -566,20 +567,31 @@ sub restamped "$used|$free", 'a backend terminated while it owns a page gives the page back'); -# A free pass paused after its claim of a slot: a second pass frees nothing, -# and the count is right once the first goes on. +# A free pass paused after its claim of a slot holds the pass's lock: a +# second pass waits at it and frees nothing meanwhile, and the count is right +# once the first goes on (bugs row 309's one pass at a time). $kept = paged($LARGE); $used = delta(0, 0, 0); -hold('doltgres-delta-claimed'); my $passer = $node->background_psql('postgres', on_error_stop => 0); +$passer->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-delta-claimed', 'wait')" +); start($passer, "SELECT doltgres_test_delta(4, $kept, 0)"); $node->wait_for_event('client backend', 'doltgres-delta-claimed'); -delta(23, 0, 0); +my $second_pass = $node->background_psql('postgres', on_error_stop => 0); +start($second_pass, 'SELECT doltgres_test_delta(23, 0, 0)'); +$node->poll_query_until('postgres', + "SELECT wait_event_type = 'LWLock' FROM pg_stat_activity WHERE pid = $second_pass->{pid}" +) or die 'the second free pass never waited at the first'; is(delta(0, 0, 0), - $used, "a second free pass at a pass's pause frees nothing"); -release('doltgres-delta-claimed'); + $used, "a second free pass at a pass's pause waits and frees nothing"); +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('doltgres-delta-claimed')"); is(ended($passer), '0', 'a paused free pass goes on when woken'); +isnt(ended($second_pass), undef, + 'the waiting free pass goes on once the first ends'); leave($passer); +leave($second_pass); is( delta(0, 0, 0), $used - $BLOCK, 'a free pass that was paused frees what it claimed'); @@ -1350,4 +1362,87 @@ sub raised_records ok($below->($unlinking, $before_abort + int(($aborted_filled - $before_abort) / 2)), "an aborted table's storage gives its delta's pages back at the abort"); + +# Bugs row 229's wait: a writer whose first write into a table finds room for +# its memtable's first page and none for its version waits asleep at the +# budget, and a cancel ends it with Postgres' 57014. Spill's relieved spills +# nothing for it, and its walk closes the memtable the refused try left, a +# free that ended the wait at once, so the writer tried again forever and +# never reached the sleep a cancel ends. A write to another table first +# gives the commit status table room for the writer's entry; then the delta is +# filled with test pages of every size but one free 1 KB slot, in a block +# other pages keep. +$node->safe_psql('postgres', 'DROP TABLE kilobytes'); +$node->append_conf('postgresql.conf', 'delta_memory = 1MB'); +$node->restart; +$node->safe_psql( + 'postgres', q{ +CREATE TABLE first_write (k int4, v text); +CREATE TABLE status_warmed (k int4); +SELECT doltgres_test_write('status_warmed', true, 'INSERT INTO status_warmed VALUES (1)'); +}); +my $slot = paged($PAGE); +my @sizes = ($LARGE, map { (8 << 32) | ((1024 << $_) - 64) } 0 .. 5); +my @filled; +for (1 .. 2) +{ + delta(23, 0, 0); + for my $size (@sizes) + { + my $record = record(); + push @filled, $record if delta(20, $record, $size) > 0; + } +} +delta(4, $slot, 0); +ok($node->poll_query_until('postgres', "SELECT doltgres_test_delta(10, $PAGE, 1) = 1"), + "the delta frees the one slot a test page left"); +is(join(',', map { delta(10, $_, $_ == $PAGE ? 2 : 1) } @sizes), + '0,0,0,0,0,0,0', 'the delta has no other free slot'); +my $first_writer = $node->background_psql('postgres', on_error_stop => 0); +$first_writer->query_safe('\set VERBOSITY verbose'); +start($first_writer, + "SELECT doltgres_test_write('first_write', true, 'INSERT INTO first_write VALUES (1, repeat(''x'', 2000))')"); +ok( $node->poll_query_until( + 'postgres', + "SELECT count(*) = 1 FROM pg_stat_activity WHERE wait_event = 'versioned delta budget'" + ), + "a first write with room for its memtable's first page alone waits asleep at the budget"); +$node->safe_psql('postgres', "SELECT pg_cancel_backend($first_writer->{pid})"); +awaited($first_writer, \$first_writer->{stderr}, qr/ERROR:/); +like($first_writer->{stderr}, + qr/ERROR: 57014: canceling statement due to user request/, + 'a cancel ends a wait at a full delta with 57014'); +delta(21, $_, 0) for @filled; +leave($first_writer); + +# Bugs row 355: a process answers a repeat of an open the directory admitted +# from its own table, for the same name at the same epoch only. A repeat +# reads what the first open read; once the page is retired and its slot freed +# and taken again, the old name is refused and the new one reads the new page. +my $cached = record(); +my $cached_name = delta(1, $cached, $PAGE); +delta(2, $cached, 41); +my $cacher = $node->background_psql('postgres', on_error_stop => 0); +is( $cacher->query_safe("SELECT doltgres_test_delta(27, 0, $cached_name)") + . ' ' + . $cacher->query_safe("SELECT doltgres_test_delta(27, 0, $cached_name)"), + '41 41', + 'a repeat of an admitted open in one process reads what the first open read'); +my $free_before = delta(17, 0, 0); +delta(4, $cached, 0); +delta(23, 0, 0); +is(delta(17, 0, 0), $free_before + 1, "the cached page's slot is freed"); +$cacher->query("SELECT doltgres_test_delta(27, 0, $cached_name)"); +like( + $cacher->{stderr}, + qr/a name of the delta names no page its hold reaches/, + "the process's open of the freed page's name is refused"); +$cacher->{stderr} = ''; +my $taker = record(); +my $taker_name = delta(1, $taker, $PAGE); +delta(2, $taker, 42); +is( $cacher->query_safe("SELECT doltgres_test_delta(27, 0, $taker_name)"), + '42', 'and its open of the page that took the slot reads the new page'); +$cacher->quit; + done_testing(); diff --git a/src/doltgres/test/t/036_parallel.pl b/src/doltgres/test/t/036_parallel.pl index 1c986957163c5..8bcfcdd92661a 100644 --- a/src/doltgres/test/t/036_parallel.pl +++ b/src/doltgres/test/t/036_parallel.pl @@ -149,5 +149,63 @@ sub parallel ); $before->quit; +# Bugs row 309's parallel units: a parallel query's scan of a table whose rows +# are all in the delta, run by the leader and two workers, reads the delta's +# run at most once in each of them, at 2,000 rows and at 5,000, where each +# unit read and sorted the whole run; it gives every row once; and the scan +# rescanned on a nested loop's inner side for each of three outer rows gives +# every row each time. +my $parallel_on = q{ +SET parallel_setup_cost = 0; +SET parallel_tuple_cost = 0; +SET min_parallel_table_scan_size = 0; +SET max_parallel_workers_per_gather = 2; +SET enable_hashjoin = off; +SET enable_mergejoin = off; +SET enable_material = off; +}; +$node->safe_psql( + 'postgres', q{ +CREATE EXTENSION injection_points; +}); +foreach my $rows (2000, 5000) +{ + my ($code, $out, $err) = $node->psql( + 'postgres', qq{ +BEGIN; +$parallel_on +CREATE TABLE units_$rows (a int4, b text) USING heap; +ALTER TABLE units_$rows SET (parallel_workers = 2); +INSERT INTO units_$rows SELECT n, repeat('u', 34) FROM generate_series(1, $rows) n; +ANALYZE units_$rows; +SELECT injection_points_attach('doltgres-scan-run', 'notice'); +SELECT count(*) FROM units_$rows; +SELECT injection_points_detach('doltgres-scan-run'); +\\echo ONCE +SELECT count(*), count(DISTINCT a), sum(a) FROM units_$rows; +\\echo PLAN +EXPLAIN (COSTS OFF) SELECT count(*) FROM (SELECT n FROM generate_series(1, 3) n WHERE random() >= 0) o, units_$rows u WHERE u.a > 0; +\\echo RESCANNED +SELECT count(*) FROM (SELECT n FROM generate_series(1, 3) n WHERE random() >= 0) o, units_$rows u WHERE u.a > 0; +ROLLBACK; +}); + my $runs = () = $err =~ /doltgres-scan-run/g; + ok( $runs >= 1 && $runs <= 3, + "a parallel scan of $rows delta rows reads the delta's run at most once in each process ($runs)" + ); + is( ($out =~ /ONCE\n(\S+)\n/)[0], + "$rows|$rows|" . ($rows * ($rows + 1) / 2), + "a parallel scan of $rows delta rows gives every row once"); + like( + ($out =~ /PLAN\n(.*)RESCANNED/s)[0], + qr/Nested Loop.*Gather.*Parallel Seq Scan on units_$rows/s, + "the rescanned plan has the parallel scan of $rows rows on a nested loop's inner side" + ); + is( ($out =~ /RESCANNED\n(\d+)/)[0], + 3 * $rows, + "a parallel scan of $rows delta rows rescanned for each outer row gives every row each time" + ); +} + $node->stop; done_testing(); diff --git a/src/doltgres/test/t/041_database.pl b/src/doltgres/test/t/041_database.pl index bbe71897a2845..d72ef8eadae81 100644 --- a/src/doltgres/test/t/041_database.pl +++ b/src/doltgres/test/t/041_database.pl @@ -431,5 +431,265 @@ sub files } $rekeyed->stop; +# Bugs row 99: DROP TABLESPACE of a location a fork table left, moved out by +# SET TABLESPACE or dropped, empties the fork's packs out of it into the +# database's own directory and removes the tablespace's directory of them, so +# the tablespace drops as in Postgres, after the checkpoint that removes a +# dropped table's files, and the moved table reads every row. +$node->safe_psql('postgres', 'CREATE EXTENSION IF NOT EXISTS doltgres_test'); +for my $left ('moved', 'dropped') +{ + $node->safe_psql('postgres', "CREATE TABLESPACE left_$left LOCATION ''"); + my $space = $node->safe_psql('postgres', + "SELECT oid FROM pg_tablespace WHERE spcname = 'left_$left'"); + $node->safe_psql('postgres', + "CREATE TABLE in_$left (k int4 PRIMARY KEY, v text) TABLESPACE left_$left; INSERT INTO in_$left SELECT g, repeat('v', 40) || g FROM generate_series(1, 500) g" + ); + $node->safe_psql('postgres', + "SELECT doltgres_test_fold('in_$left'); CHECKPOINT"); + my @packs = glob("$pgdata/pg_tblspc/$space/PG_*/pg_versioned/*/*.pack"); + ok(@packs > 0, + "a fold of a table in tablespace left_$left writes its packs there"); + $node->safe_psql('postgres', + $left eq 'moved' + ? "ALTER TABLE in_$left SET TABLESPACE pg_default" + : "DROP TABLE in_$left"); + my ($ret, $out, $err) = + $node->psql('postgres', "DROP TABLESPACE left_$left"); + is("$ret|$err", '0|', + $left eq 'moved' + ? 'DROP TABLESPACE of a location a fork table moved out of empties the fork\'s packs out of it' + : 'DROP TABLESPACE of a location a fork table left empties the fork\'s packs out of it' + ); + ok(!-e "$pgdata/pg_tblspc/$space", + "the dropped tablespace left_$left leaves no directory"); +} +is( $node->safe_psql( + 'postgres', 'SELECT count(*) || \' \' || sum(k) FROM in_moved'), + '500 125250', + "the moved table reads every row after its old tablespace drops"); + +# Bugs row 99: a base backup keeps every collection from running, so DROP +# TABLESPACE of a location that holds the fork's packs waits for the backup's +# end, where Postgres drops it at once: a cancel ends the wait with Postgres' +# error and keeps the tablespace, and a DROP waiting at pg_backup_stop then +# drops it. +$node->safe_psql('postgres', + "CREATE TABLESPACE backed LOCATION ''; CREATE TABLE in_backed (k int4) TABLESPACE backed; INSERT INTO in_backed SELECT generate_series(1, 100)" +); +$node->safe_psql('postgres', + "SELECT doltgres_test_fold('in_backed'); ALTER TABLE in_backed SET TABLESPACE pg_default; CHECKPOINT" +); +my $backed = $node->safe_psql('postgres', + "SELECT oid FROM pg_tablespace WHERE spcname = 'backed'"); +my $backup = $node->background_psql('postgres'); +$backup->query_safe("SELECT pg_backup_start('row99', true)"); +my $waiting = + q{SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE query = 'DROP TABLESPACE backed;' AND wait_event_type = 'Extension')}; +my $dropper = $node->background_psql('postgres', on_error_stop => 0); +$dropper->query_safe('\\set VERBOSITY verbose'); +$dropper->query_until(qr/start/, "\\echo start\nDROP TABLESPACE backed;\n"); +$node->poll_query_until('postgres', $waiting); +$node->safe_psql('postgres', + "SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE query = 'DROP TABLESPACE backed;'" +); +$dropper->query('SELECT 1'); +like( + $dropper->{stderr}, + qr/ERROR: 57014: canceling statement due to user request/, + "a cancel ends DROP TABLESPACE's wait for a base backup with Postgres' error" +); +is( $node->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_tablespace WHERE spcname = 'backed'"), + '1', + "a canceled DROP TABLESPACE keeps the tablespace"); +$dropper->{stderr} = ''; +$dropper->query_until(qr/again/, "\\echo again\nDROP TABLESPACE backed;\n"); +$node->poll_query_until('postgres', $waiting); +$backup->query('SELECT 1 FROM pg_backup_stop()'); +$dropper->query('SELECT 1'); +is( $dropper->{stderr} + . ( + $node->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_tablespace WHERE spcname = 'backed'")), + '0', + "DROP TABLESPACE during a base backup drops the tablespace once pg_backup_stop ends the backup" +); +ok(!-e "$pgdata/pg_tblspc/$backed", + "the tablespace dropped after the backup leaves no directory"); +$_->quit for $backup, $dropper; + +# Bugs row 99 on a hot standby: its replay of DROP TABLESPACE empties its own +# packs out of the tablespace, so the replay removes the tablespace's +# directories and the moved table reads every row; and when a reader holds +# the packs through the replay, Postgres' replay logs what it couldn't +# remove, and the collector's sweep removes the fork's directory of packs +# once the reader ends, with no restart. +$node->backup('spaced'); +my $spaced = PostgreSQL::Test::Cluster->new('spaced_standby'); +$spaced->init_from_backup($node, 'spaced', has_streaming => 1); +$spaced->append_conf('postgresql.conf', 'max_standby_streaming_delay = 1s'); +$spaced->start; +my $spaced_dir = $spaced->data_dir; +# standby_space makes a tablespace with a table whose fold the standby +# replays into packs of its own there, and returns the tablespace's id. +sub standby_space +{ + my ($held) = @_; + $node->safe_psql('postgres', + "CREATE TABLESPACE standby_$held LOCATION ''; CREATE TABLE on_$held (k int4 PRIMARY KEY, v text) TABLESPACE standby_$held; INSERT INTO on_$held SELECT g, repeat('w', 40) || g FROM generate_series(1, 300) g" + ); + $node->safe_psql('postgres', + "SELECT doltgres_test_fold('on_$held'); CHECKPOINT"); + $node->wait_for_replay_catchup($spaced); + my $space = $node->safe_psql('postgres', + "SELECT oid FROM pg_tablespace WHERE spcname = 'standby_$held'"); + my $made = time + $PostgreSQL::Test::Utils::timeout_default; + sleep 1 + while !(() = + glob("$spaced_dir/pg_tblspc/$space/PG_*/pg_versioned/*/*.pack")) + && time < $made; + return $space; +} + +# standby_moved moves a tablespace's table out to pg_default and drops the +# tablespace, each replayed on the standby. +sub standby_moved +{ + my ($held) = @_; + $node->safe_psql('postgres', + "ALTER TABLE on_$held SET TABLESPACE pg_default; CHECKPOINT"); + $node->wait_for_replay_catchup($spaced); + $node->safe_psql('postgres', "DROP TABLESPACE standby_$held"); + $node->wait_for_replay_catchup($spaced); +} +my $free = standby_space('free'); +standby_moved('free'); +ok( !-e "$spaced_dir/pg_tblspc/$free", + "a hot standby's replay of DROP TABLESPACE removes the tablespace's directories with the fork's packs" +); +is( $spaced->safe_psql( + 'postgres', 'SELECT count(*) || \' \' || sum(k) FROM on_free'), + '300 45150', + "the standby reads every row of the table moved out of the dropped tablespace" +); +my $read = standby_space('read'); +$spaced->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-shared-reading', 'wait')"); +my $holder = $spaced->background_psql('postgres', on_error_stop => 0); +$holder->query_until(qr/reading/, + "\\echo reading\nSELECT doltgres_test_shared(11, 0, 0);\n"); +$spaced->wait_for_event('client backend', 'doltgres-shared-reading'); +standby_moved('read'); +$spaced->psql('postgres', + "SELECT injection_points_wakeup('doltgres-shared-reading')"); +$holder->query('SELECT 1'); +like( + $holder->{stderr}, + qr/ERROR: canceling statement due to conflict with recovery/, + "a standby reader that holds the packs through the replay of DROP TABLESPACE is canceled as Postgres' replay cancels a tablespace's users" +); +ok( !-e "$spaced_dir/pg_tblspc/$read", + "and the replay's retry after the cancel empties the fork's packs out of the tablespace and removes its directories, with no restart" +); +$spaced->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-shared-reading')"); +$holder->quit; +is( $spaced->safe_psql( + 'postgres', 'SELECT count(*) || \' \' || sum(k) FROM on_read'), + '300 45150', + "the standby reads every row of the table moved out of that tablespace"); +$spaced->stop; + +# Bugs row 99 on a standby without hot standby: no collector worker runs to +# empty its packs at the replay of DROP TABLESPACE, so Postgres' replay logs +# what it couldn't remove after leaving the dropped mark, and once the +# standby starts again with hot standby, after a crash, the collector's +# sweep empties the fork's packs out of the marked tablespace. +$node->backup('cold'); +my $cold = PostgreSQL::Test::Cluster->new('cold_standby'); +$cold->init_from_backup($node, 'cold', has_streaming => 1); +$cold->append_conf('postgresql.conf', 'hot_standby = off'); +$cold->start; +my $cold_dir = $cold->data_dir; +# cold_replayed waits until the standby without hot standby has replayed the +# primary's WAL to its current end, read from the primary's view of it. +sub cold_replayed +{ + my $lsn = $node->lsn('insert'); + $node->poll_query_until('postgres', + "SELECT replay_lsn >= '$lsn' FROM pg_stat_replication WHERE application_name = 'cold_standby'" + ) or die "the standby without hot standby didn't replay to $lsn"; +} +$node->safe_psql('postgres', + "CREATE TABLESPACE cold_kept LOCATION ''; CREATE TABLE on_cold (k int4 PRIMARY KEY, v text); INSERT INTO on_cold SELECT g, repeat('c', 40) || g FROM generate_series(1, 300) g" +); +$node->safe_psql('postgres', + "SELECT doltgres_test_fold('on_cold'); ALTER TABLE on_cold SET TABLESPACE cold_kept; CHECKPOINT" +); +my $cold_space = $node->safe_psql('postgres', + "SELECT oid FROM pg_tablespace WHERE spcname = 'cold_kept'"); +cold_replayed(); +my @cold_packs = + glob("$cold_dir/pg_tblspc/$cold_space/PG_*/pg_versioned/*/*.pack"); +ok( @cold_packs > 0, + "a standby without hot standby replays a move into a tablespace as packs of its own there" +); +$node->safe_psql('postgres', + 'ALTER TABLE on_cold SET TABLESPACE pg_default; CHECKPOINT'); +$node->safe_psql('postgres', 'DROP TABLESPACE cold_kept'); +cold_replayed(); +my @cold_left = glob("$cold_dir/pg_tblspc/$cold_space/PG_*/pg_versioned"); +ok( @cold_left == 1, + "a standby without hot standby keeps the fork's packs through its replay of DROP TABLESPACE" +); +ok(-e "$cold_left[0]/dropped", + "and its replay leaves the dropped mark on them"); +$cold->stop('immediate'); +$cold->append_conf('postgresql.conf', 'hot_standby = on'); +$cold->start; +my $cold_swept = time + $PostgreSQL::Test::Utils::timeout_default; +sleep 1 while -e $cold_left[0] && time < $cold_swept; +ok( !-e $cold_left[0], + "the collector's sweep empties the fork's packs out of a tablespace whose catalog row is gone" +); +is( $cold->safe_psql( + 'postgres', 'SELECT count(*) || \' \' || sum(k) FROM on_cold'), + '300 45150', + "the swept standby reads every row of the table moved out of the tablespace" +); +$cold->stop; + +# Bugs row 99: a table another session creates in the tablespace while DROP +# TABLESPACE waits for a base backup stops the DROP's emptying at its next +# round, before any of the new table's packs could be moved, so Postgres +# refuses the DROP as not empty, as it would with no wait. +$node->safe_psql('postgres', + "CREATE TABLESPACE raced LOCATION ''; CREATE TABLE in_raced (k int4) TABLESPACE raced; INSERT INTO in_raced SELECT generate_series(1, 100)" +); +$node->safe_psql('postgres', + "SELECT doltgres_test_fold('in_raced'); ALTER TABLE in_raced SET TABLESPACE pg_default; CHECKPOINT" +); +my $raced_backup = $node->background_psql('postgres'); +$raced_backup->query_safe("SELECT pg_backup_start('raced', true)"); +my $racing = $node->background_psql('postgres', on_error_stop => 0); +$racing->query_safe('\\set VERBOSITY verbose'); +$racing->query_until(qr/raced/, "\\echo raced\nDROP TABLESPACE raced;\n"); +$node->poll_query_until('postgres', + q{SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE query = 'DROP TABLESPACE raced;' AND wait_event_type = 'Extension')} +); +$node->safe_psql('postgres', + 'CREATE TABLE in_raced_late (k int4) TABLESPACE raced'); +$racing->query('SELECT 1'); +like( + $racing->{stderr}, + qr/ERROR: 55000: tablespace "raced" is not empty/, + "a table created in the tablespace while DROP TABLESPACE waits stops its emptying, and Postgres refuses the DROP" +); +$raced_backup->query('SELECT 1 FROM pg_backup_stop()'); +$_->quit for $racing, $raced_backup; + $node->stop; done_testing(); diff --git a/src/doltgres/test/t/044_removal.pl b/src/doltgres/test/t/044_removal.pl index c9e03f0709693..756bff95195cc 100644 --- a/src/doltgres/test/t/044_removal.pl +++ b/src/doltgres/test/t/044_removal.pl @@ -83,5 +83,31 @@ "a backend terminated inside its transaction takes its rows back at its exit's abort" ); +# Bugs row 309: an abort's take-back of twenty rows of one table finds the +# table's state once, not once for each row it wrote. +SKIP: +{ + skip 'injection points are not enabled', 1 + if $ENV{enable_injection_points} ne 'yes' + || !$node->check_extension('injection_points'); + $node->safe_psql( + 'postgres', q{ +CREATE EXTENSION injection_points; +CREATE TABLE source_twenty AS SELECT generate_series(1, 20) AS a; +CREATE TABLE found_twenty (a int4); +}); + my ($code, $out, $err) = $node->psql( + 'postgres', q{ +SELECT injection_points_set_local(); +SELECT injection_points_attach('doltgres-state-found', 'notice'); +BEGIN; +SELECT doltgres_test_insert('found_twenty', 'source_twenty', false, 0); +ROLLBACK; +}); + my $found = () = $err =~ /doltgres-state-found/g; + is($found, 1, + "an abort's take-back of twenty rows finds their table's state once"); +} + $node->stop; done_testing(); diff --git a/src/doltgres/test/t/045_btree.pl b/src/doltgres/test/t/045_btree.pl index 6d538ff4b6933..824af99f90287 100644 --- a/src/doltgres/test/t/045_btree.pl +++ b/src/doltgres/test/t/045_btree.pl @@ -2481,5 +2481,130 @@ sub parallel_ordered 't|t', "an answer of none leaves the caller's handle and datum as they were"); +# Bugs row 313: a concurrent build's validation forms no key of a row its +# build's snapshot saw, which the build indexed, so a REINDEX CONCURRENTLY runs +# an indexed row's expression once, as heap's validation does: the regress +# suite's concur_reindex_ddl sequence gives the reference's one NOTICE and its +# two errors. +my (undef, undef, $ddl_err) = $node->psql( + 'postgres', q{ +CREATE TABLE concur_reindex_ddl_tab (i int); +INSERT INTO concur_reindex_ddl_tab VALUES (1); +CREATE FUNCTION concur_reindex_ddl_f() RETURNS void LANGUAGE plpgsql AS + 'begin DROP INDEX IF EXISTS public.concur_reindex_ddl_ind2; end'; +CREATE FUNCTION concur_reindex_ddl_im(int) RETURNS int + LANGUAGE plpgsql IMMUTABLE AS + 'begin perform public.concur_reindex_ddl_f(); return $1; end'; +CREATE INDEX concur_reindex_ddl_ind1 ON + concur_reindex_ddl_tab (concur_reindex_ddl_im(i)); +CREATE INDEX concur_reindex_ddl_ind2 ON concur_reindex_ddl_tab (i); +REINDEX TABLE CONCURRENTLY concur_reindex_ddl_tab; +REINDEX INDEX CONCURRENTLY concur_reindex_ddl_ind2; +REINDEX INDEX CONCURRENTLY concur_reindex_ddl_ind1; +REINDEX INDEX CONCURRENTLY concur_reindex_ddl_ind2; +}, + on_error_stop => 0, + extra_params => [ '-v', 'VERBOSITY=verbose' ]); +is( join("\n", $ddl_err =~ /((?:NOTICE|ERROR): \w{5}: [^\n]*)/g), + join("\n", + 'NOTICE: 00000: index "concur_reindex_ddl_ind2" does not exist, skipping', + 'ERROR: 55006: cannot DROP INDEX "concur_reindex_ddl_ind2" because it is being used by active queries in this session', + 'ERROR: 42P01: relation "concur_reindex_ddl_ind2" does not exist'), + "a REINDEX CONCURRENTLY computes an indexed row's expression once, as heap's validation does" +); + +# Bugs row 313: CREATE INDEX CONCURRENTLY and REINDEX CONCURRENTLY of a fork +# btree expression index while another session, which held its lock across the +# build, inserts while the command waits for writers before its validation: +# the index is valid and holds every row once. The index's expression waits on +# an advisory lock at one row, so the writer's lock comes after the build's +# first wait. +$node->safe_psql( + 'postgres', q{ +CREATE FUNCTION validated_at(k int) RETURNS int IMMUTABLE LANGUAGE plpgsql + AS $$ BEGIN IF k = 2 THEN PERFORM pg_advisory_xact_lock_shared(45); END IF; RETURN k; END $$; +CREATE TABLE validated (k int); +INSERT INTO validated SELECT generate_series(1, 3); +}); +my $validated_locker = $node->background_psql('postgres'); +my $validated_builder = $node->background_psql('postgres'); +my $validated_writer = $node->background_psql('postgres'); +my $validated_first = 4; +foreach my $statement ( + 'CREATE INDEX CONCURRENTLY validated_k ON validated ((validated_at(k)))', + 'REINDEX INDEX CONCURRENTLY validated_k') +{ + $validated_locker->query_safe('SELECT pg_advisory_lock(45)'); + $validated_builder->query_until(qr/start/, "\\echo start\n$statement;\n"); + $node->poll_query_until('postgres', + q{SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND query ~ 'CONCURRENTLY')} + ); + $validated_writer->query_safe( + 'BEGIN; LOCK TABLE validated IN ROW EXCLUSIVE MODE'); + $validated_locker->query_safe('SELECT pg_advisory_unlock(45)'); + $node->poll_query_until('postgres', + q{SELECT EXISTS (SELECT 1 FROM pg_stat_progress_create_index WHERE phase = 'waiting for writers before validation')} + ); + my $last = $validated_first + 1; + $validated_writer->query_safe( + "INSERT INTO validated SELECT generate_series($validated_first, $last); COMMIT" + ); + $validated_builder->query_safe('SELECT 1'); + is( $node->safe_psql( + 'postgres', + "SET enable_seqscan = off; SET enable_bitmapscan = off; SELECT (SELECT indisvalid FROM pg_index WHERE indexrelid = 'validated_k'::regclass) || ' ' || string_agg(k::text, ',' ORDER BY k) FROM validated WHERE validated_at(k) > 0" + ), + 'true ' . join(',', 1 .. $last), + "$statement while another session inserts before its validation leaves a valid index holding every row once" + ); + $validated_first = $last + 1; +} +$_->quit for $validated_locker, $validated_builder, $validated_writer; + $node->stop; + +# P2's index build and index move: a build whose snapshot predates another +# transaction's committed insert of rows past a 4 MB delta_memory carries their +# late entries to spill files under the writer's xid and completes; held +# unfolded by an older snapshot, the index then moves into a tablespace with +# those entries carried again, never waiting at the budget, and its storage +# holds an entry for every row after the move and after a crash. +my $moving_index = PostgreSQL::Test::Cluster->new('index_move'); +$moving_index->init; +$moving_index->append_conf('postgresql.conf', + "delta_memory = 4MB\nfold_threshold = 1GB\nfold_timeout = 3600s\ncollector_naptime = 3600s\nallow_in_place_tablespaces = on" +); +$moving_index->start; +$moving_index->safe_psql( + 'postgres', q{ + CREATE EXTENSION doltgres_test; + CREATE TABLESPACE index_moved LOCATION ''; + CREATE TABLE indexed (id int PRIMARY KEY, name text); + CREATE INDEX indexed_name ON indexed (name); +}); +my $entries = sub { + scalar(() = $moving_index->safe_psql('postgres', q{SELECT doltgres_test_index_entries('indexed_name')}) =~ /^put$/mg); +}; +my $index_older = $moving_index->background_psql('postgres'); +$index_older->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT 1'); +my $index_builder = $moving_index->background_psql('postgres'); +$index_builder->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT 1'); +$moving_index->safe_psql('postgres', + q{SELECT doltgres_test_write('indexed', true, 'INSERT INTO indexed SELECT g, repeat(md5(g::text), 31) FROM generate_series(1, 10000) g')}, + timeout => 600); +$index_builder->query_safe(q{SELECT doltgres_test_index_build('indexed', 'indexed_name')}); +$index_builder->query_safe('COMMIT'); +$index_builder->quit; +is($entries->(), 10000, "a build at a 4 MB delta_memory carries every late entry of another transaction's rows"); +$moving_index->safe_psql('postgres', + q{SELECT doltgres_test_index_routine('indexed_name'); SELECT doltgres_test_index_move('indexed_name', 'index_moved')}, + timeout => 600); +is($entries->(), 10000, "an index move held unfolded carries every entry of another transaction past delta_memory"); +$index_older->query_safe('COMMIT'); +$index_older->quit; +$moving_index->stop('immediate'); +$moving_index->start; +is($entries->(), 10000, 'and the moved entries survive a crash'); +$moving_index->stop; + done_testing(); diff --git a/src/doltgres/test/t/046_update.pl b/src/doltgres/test/t/046_update.pl index ae0999047d526..128170d9e5dc5 100644 --- a/src/doltgres/test/t/046_update.pl +++ b/src/doltgres/test/t/046_update.pl @@ -1073,4 +1073,200 @@ sub both "an update carries its old row's lock to the new row's xmax as heap's does, with columns ($columns)"); } + +# A primary key WITHOUT OVERLAPS is an exclusion constraint's, so an insert or +# an update that writes an existing row's key is refused with the exclusion +# violation, 23P01, as heap's exclusion check refuses it (bugs row 317), not +# with a unique index's 23505, the existing row folded into the base. On a +# node of its own, with the fold launcher held, no fold worker left running, +# and no other process's xmin or xid holding the horizon, the case's own fold +# installs that base. +my $overlap = PostgreSQL::Test::Cluster->new('overlap'); +$overlap->init; +$overlap->start; +$overlap->safe_psql('postgres', + "CREATE EXTENSION injection_points; CREATE EXTENSION doltgres_test; SELECT injection_points_attach('doltgres-fold-launch', 'wait')" +); +$overlap->poll_query_until('postgres', + "SELECT count(*) = 1 FROM pg_stat_activity WHERE wait_event = 'doltgres-fold-launch'" +) or die 'the fold launcher is not held'; +$overlap->safe_psql( + 'postgres', q{ +CREATE TABLE overlapped (id int4range, valid_at daterange, CONSTRAINT overlapped_pkey PRIMARY KEY (id, valid_at WITHOUT OVERLAPS)); +INSERT INTO overlapped VALUES ('[1,2)', daterange('2000-01-01', '2000-02-15')), ('[1,2)', daterange('2001-01-01', '2002-01-01')); +}); +$overlap->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND (backend_type = 'fold worker' OR backend_xmin IS NOT NULL OR backend_xid IS NOT NULL)" +) or die 'a process holds the fold back'; +is( $overlap->safe_psql( + 'postgres', "SELECT doltgres_test_fold('overlapped')"), + 't', + 'a WITHOUT OVERLAPS primary key\'s table folds its rows into its base'); +my @refused; +for my $statement ( + "INSERT INTO overlapped VALUES ('[1,2)', daterange('2000-01-01', '2000-02-15'))", + "UPDATE overlapped SET valid_at = daterange('2000-01-01', '2000-04-01') WHERE id = '[1,2)'" + ) +{ + my (undef, undef, $err) = + $overlap->psql('postgres', "\\set VERBOSITY verbose\n$statement"); + push @refused, + join("\n", + grep { /^(?:psql::\d+: )?(?:ERROR|DETAIL):/ } split /\n/, + $err); +} +is_deeply( + \@refused, + [ + qq{psql::2: ERROR: 23P01: conflicting key value violates exclusion constraint "overlapped_pkey"\nDETAIL: Key (id, valid_at)=([1,2), [2000-01-01,2000-02-15)) conflicts with existing key (id, valid_at)=([1,2), [2000-01-01,2000-02-15)).}, + qq{psql::2: ERROR: 23P01: conflicting key value violates exclusion constraint "overlapped_pkey"\nDETAIL: Key (id, valid_at)=([1,2), [2000-01-01,2000-04-01)) conflicts with existing key (id, valid_at)=([1,2), [2000-01-01,2000-04-01)).} + ], + "an insert or an update of a WITHOUT OVERLAPS primary key's existing key is refused as heap's exclusion check refuses it" +); +$overlap->stop; + +# An update's flagged take of its replaced row's block, whose read a spill of the block's running writer moves +# after it, finishes with the block's spilled entry, and an error at the block's read again fails only the update +# (bugs row 342). +my $flagging = PostgreSQL::Test::Cluster->new('flagging'); +$flagging->init(allows_streaming => 1); +$flagging->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\nautovacuum = off"); +$flagging->start; +$flagging->safe_psql('postgres', + 'CREATE EXTENSION injection_points; CREATE EXTENSION doltgres_test'); +$flagging->backup('flagging_backup'); +my $flagging_standby = PostgreSQL::Test::Cluster->new('flagging_standby'); +$flagging_standby->init_from_backup($flagging, 'flagging_backup', + has_streaming => 1); +$flagging_standby->start; + +# flagging_made makes a table whose block 0, nearly full and left for block 1, holds a folded row, so an update to +# 3000 bytes moves it to block 1. +sub flagging_made +{ + my ($table) = @_; + $flagging->safe_psql( + 'postgres', qq{ +CREATE TABLE $table (k int4, v text); +ALTER TABLE $table ALTER COLUMN v SET STORAGE PLAIN; +INSERT INTO $table VALUES (1, repeat('v', 2500)); +INSERT INTO $table VALUES (4, repeat('v', 4500)); +INSERT INTO $table VALUES (3, repeat('v', 1500)); +SELECT doltgres_test_fold('$table'); +}); +} + +# flagging_held runs the update of a table's row 1 held after its flagged read, while the writer spills its takes on +# block 0, its second turn's flagged read erring when error is set; it returns the update's errors. +sub flagging_held +{ + my ($table, $error) = @_; + my $writer = $flagging->background_psql('postgres'); + $writer->query_safe( + "BEGIN; INSERT INTO $table VALUES (2, repeat('v', 600))"); + my $updater = $flagging->background_psql('postgres', on_error_stop => 0); + $updater->query_safe('SELECT injection_points_set_local()'); + $updater->query_safe( + "SELECT injection_points_attach('doltgres-insert-flagged-read', 'wait')" + ); + $updater->query_until(qr/started/, + "\\echo started\nUPDATE $table SET v = repeat('w', 3000) WHERE k = 1;\n" + ); + $flagging->wait_for_event('client backend', + 'doltgres-insert-flagged-read'); + $writer->query_safe('SELECT doltgres_test_delta(61, 0, 0)') for 1 .. 4; + $flagging->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-insert-flagged-read')"); + $flagging->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-insert-flagged-read', 'error')" + ) if $error; + $flagging->safe_psql('postgres', + "SELECT injection_points_wakeup('doltgres-insert-flagged-read')"); + $updater->query('SELECT 1'); + $flagging->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-insert-flagged-read')") + if $error; + my $errors = $updater->{stderr}; + $updater->quit; + $writer->query_safe('COMMIT'); + $writer->quit; + return $errors; +} +my $flagging_entry = q{SELECT doltgres_test_free_space('%s', 0)}; + +flagging_made('flagged_spilled'); +flagging_held('flagged_spilled', 0); +flagging_made('flagged_plain'); +$flagging->safe_psql( + 'postgres', q{ +INSERT INTO flagged_plain VALUES (2, repeat('v', 600)); +UPDATE flagged_plain SET v = repeat('w', 3000) WHERE k = 1; +}); +is( $flagging->safe_psql( + 'postgres', sprintf($flagging_entry, 'flagged_spilled')), + $flagging->safe_psql( + 'postgres', sprintf($flagging_entry, 'flagged_plain')), + "an update's flagged take after a spill moved its block's takes leaves the entry an unspilled update leaves" +); + +flagging_made('flagged_erred'); +my $erred = flagging_held('flagged_erred', 1); +like( + $erred, + qr/doltgres-insert-flagged-read/, + "an error at the flagged block's read again fails the update"); +is( $flagging->safe_psql( + 'postgres', q{SELECT length(v) FROM flagged_erred WHERE k = 1}), + '2500', + "the failed update leaves its row as it was"); +$flagging->safe_psql('postgres', + q{UPDATE flagged_erred SET v = repeat('w', 3000) WHERE k = 1}); +flagging_made('flagged_undone'); +$flagging->safe_psql( + 'postgres', q{ +INSERT INTO flagged_undone VALUES (2, repeat('v', 600)); +BEGIN; +UPDATE flagged_undone SET v = repeat('w', 3000) WHERE k = 1; +ROLLBACK; +UPDATE flagged_undone SET v = repeat('w', 3000) WHERE k = 1; +}); +is( $flagging->safe_psql( + 'postgres', sprintf($flagging_entry, 'flagged_erred')), + $flagging->safe_psql( + 'postgres', sprintf($flagging_entry, 'flagged_undone')), + "a later update after the failed one leaves the entry a rolled-back update leaves" +); + +$flagging->wait_for_catchup($flagging_standby); +foreach my $table ('flagged_spilled', 'flagged_plain', 'flagged_erred', + 'flagged_undone') +{ + is( $flagging_standby->safe_psql( + 'postgres', sprintf($flagging_entry, $table)), + $flagging->safe_psql('postgres', sprintf($flagging_entry, $table)), + "the standby's entry of $table matches the primary's"); +} +$flagging->safe_psql('postgres', 'CHECKPOINT'); +$flagging->safe_psql('postgres', + q{UPDATE flagged_plain SET v = repeat('x', 10) WHERE k = 2}); +$flagging->safe_psql('postgres', + q{UPDATE flagged_spilled SET v = repeat('x', 10) WHERE k = 2}); +$flagging->stop('immediate'); +$flagging->start; +is( $flagging->safe_psql( + 'postgres', sprintf($flagging_entry, 'flagged_spilled')), + $flagging->safe_psql( + 'postgres', sprintf($flagging_entry, 'flagged_plain')), + "after a checkpoint and a crash the spilled update's entry matches the unspilled one's" +); +$flagging->wait_for_catchup($flagging_standby); +is( $flagging_standby->safe_psql( + 'postgres', sprintf($flagging_entry, 'flagged_spilled')), + $flagging->safe_psql( + 'postgres', sprintf($flagging_entry, 'flagged_spilled')), + "and the standby's matches the primary's after the crash"); +$flagging_standby->stop; +$flagging->stop; + done_testing(); diff --git a/src/doltgres/test/t/047_rewrite.pl b/src/doltgres/test/t/047_rewrite.pl index 1e44f5a0242a1..eb616595c8cfe 100644 --- a/src/doltgres/test/t/047_rewrite.pl +++ b/src/doltgres/test/t/047_rewrite.pl @@ -1251,7 +1251,8 @@ sub vacuumed "a new standby snapshot reads $table as heap's twin"); is($node->safe_psql('postgres', $stamp_read), $stamp, "the move keeps the stamps of a row of $table updated to its own value"); - eval { $standby_older->query('COMMIT'); $standby_older->quit }; + eval { $standby_older->query('COMMIT') }; + $standby_older->quit; } $relocating->stop; @@ -1904,4 +1905,148 @@ sub vacuumed } $node->stop; + +# P2 (bugs row 229): a rewrite's carry never waits on other writers' versions. +# Session A holds a REPEATABLE READ snapshot from before B, so no fold applies +# B's commit; B commits about 10 MB of rows past a 4 MB delta_memory, which its +# own spills relieve; C's VACUUM FULL of the table carries B's versions to spill +# files under B's xid and completes. A still reads the table as it was, a new +# snapshot reads B's rows, and a crash keeps them. +my $budgeted = PostgreSQL::Test::Cluster->new('carry_budget'); +$budgeted->init; +# A count's parallel workers would read the table through heap, since the fork's +# routine stands in the leader alone. +$budgeted->append_conf('postgresql.conf', + "delta_memory = 4MB\nfold_threshold = 1GB\nfold_timeout = 3600s\ncollector_naptime = 3600s\nmax_parallel_workers_per_gather = 0" +); +$budgeted->start; +$budgeted->safe_psql('postgres', + 'CREATE EXTENSION injection_points; CREATE EXTENSION doltgres_test; CREATE TABLE carried (k int4 PRIMARY KEY, v text)' +); +my $carried_count = rows('carried', 'SELECT count(*) FROM %'); +my $before_carry = $budgeted->background_psql('postgres'); +$before_carry->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT 1'); +$budgeted->safe_psql('postgres', + written('carried', "INSERT INTO % SELECT g, repeat(md5(g::text), 31) FROM generate_series(1, 10000) g")); +$budgeted->safe_psql('postgres', rewritten('carried', undef, 'false'), timeout => 600); +pass('a VACUUM FULL over another transaction\'s committed rows past delta_memory, held unfolded by a snapshot, completes'); +is($before_carry->query_safe($carried_count), '0', 'the older snapshot still reads none of the committed rows'); +$before_carry->query_safe('COMMIT'); +$before_carry->quit; +is($budgeted->safe_psql('postgres', $carried_count), '10000', 'a new snapshot reads every row the carry spilled'); +$budgeted->stop('immediate'); +$budgeted->start; +is($budgeted->safe_psql('postgres', $carried_count), '10000', 'the carried rows survive a crash'); + +# A carry the server crashes in, after its files are written and before they +# are listed: after the restart, whose redo replays the carry's records into +# the new relfilenode its transaction left, the table reads as it was before +# the rewrite, and a rewrite then carries every row. +$before_carry = $budgeted->background_psql('postgres'); +$before_carry->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT 1'); +$budgeted->safe_psql('postgres', + written('carried', "INSERT INTO % SELECT g, repeat(md5(g::text), 31) FROM generate_series(10001, 20000) g")); +$budgeted->safe_psql('postgres', "SELECT injection_points_attach('doltgres-carry-written', 'wait')"); +my $cut_short = $budgeted->background_psql('postgres', on_error_stop => 0); +$cut_short->query_until(qr/start/, "\\echo start\n" . rewritten('carried', undef, 'false') . ";\n"); +$budgeted->wait_for_event('client backend', 'doltgres-carry-written'); +$budgeted->stop('immediate'); +eval { $cut_short->quit }; +eval { $before_carry->quit }; +$budgeted->start; +is($budgeted->safe_psql('postgres', $carried_count), '20000', 'the table reads as it was before the rewrite'); +$budgeted->safe_psql('postgres', rewritten('carried', undef, 'false'), timeout => 600); +is($budgeted->safe_psql('postgres', $carried_count), '20000', 'a rewrite after the restart carries every row'); +$budgeted->stop; + +# Ruling (b)'s case (iii): a moved table's base on a standby is its primary's, +# root, fold point, and read point alike, its read point no higher than its fold +# point. +my $points = PostgreSQL::Test::Cluster->new('moved_points'); +$points->init(allows_streaming => 1); +$points->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\ncollector_naptime = 3600s\nallow_in_place_tablespaces = on"); +$points->start; +$points->safe_psql('postgres', + q{CREATE EXTENSION doltgres_test; CREATE TABLESPACE points_moved LOCATION ''; CREATE TABLE pointed (k int4 PRIMARY KEY, v text)}); +$points->safe_psql('postgres', written('pointed', "INSERT INTO % SELECT g, 'r' || g FROM generate_series(1, 2000) g")); +$points->safe_psql('postgres', "SELECT doltgres_test_fold('pointed')"); +$points->backup('points_backup'); +my $points_standby = PostgreSQL::Test::Cluster->new('moved_points_standby'); +$points_standby->init_from_backup($points, 'points_backup', has_streaming => 1); +$points_standby->append_conf('postgresql.conf', 'allow_in_place_tablespaces = on'); +$points_standby->start; +$points->safe_psql('postgres', written('pointed', "INSERT INTO % SELECT g, 'r' || g FROM generate_series(2001, 2100) g")); +$points->safe_psql('postgres', "SELECT doltgres_test_move('pointed', 'points_moved')"); +$points->wait_for_catchup($points_standby); +my $primary_points = $points->safe_psql('postgres', "SELECT doltgres_test_base_points('pointed')"); +my ($root, $folded, $read) = split / /, $primary_points; +ok($root ne '' && $read <= $folded, "a moved base's read point is no higher than its fold point"); +is($points_standby->safe_psql('postgres', "SELECT doltgres_test_base_points('pointed')"), + $primary_points, "a standby's base of a moved table has its primary's root, fold point, and read point"); +$points_standby->stop; +$points->stop; + +# RevA's Q2 guard: a committed transaction's free space takes sit above those +# of a writer begun after it, which commits while a snapshot taken before its +# commit holds the table's horizon, so the move's fold keeps the takes above the +# later writer's with its rows' out-of-line values; the move lowers the moved +# base's read point below its fold point, and a rewrite before any fold carries +# every row, so the table's rows and values equal heap's twin's, and no tuple id +# is given twice. +my $guarded = PostgreSQL::Test::Cluster->new('rewrite_guard'); +$guarded->init; +$guarded->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\ncollector_naptime = 3600s\nallow_in_place_tablespaces = on\nmax_parallel_workers_per_gather = 0" +); +$guarded->start; +$guarded->safe_psql( + 'postgres', q{ +CREATE EXTENSION doltgres_test; +CREATE TABLESPACE guard_moved LOCATION ''; +CREATE TABLE guarded (k int4 PRIMARY KEY, v text); +CREATE TABLE heap_guarded (k int4 PRIMARY KEY, v text); +}); +my $long = "string_agg(md5(g::text || ':' || n::text), '') FROM generate_series(1, 120) n"; +for my $table ('guarded', 'heap_guarded') +{ + $guarded->safe_psql('postgres', + written($table, "INSERT INTO % SELECT g, 'r' || g FROM generate_series(1, 400) g")); +} +$guarded->safe_psql('postgres', "SELECT doltgres_test_fold('guarded')"); +my ($kept, $running) = map { $guarded->background_psql('postgres') } 1 .. 2; +$kept->query_safe('BEGIN; SELECT pg_current_xact_id()'); +$running->query_safe('BEGIN'); +for my $table ('guarded', 'heap_guarded') +{ + $running->query_safe(written($table, "INSERT INTO % SELECT g, 'o' || g FROM generate_series(401, 405) g")); +} +for my $table ('guarded', 'heap_guarded') +{ + $kept->query_safe(written($table, "INSERT INTO % SELECT g, (SELECT $long) FROM generate_series(406, 430) g")); +} +$kept->query_safe('COMMIT'); +$kept->quit; +my $holder = $guarded->background_psql('postgres'); +$holder->query_safe('BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT 1'); +$running->query_safe('COMMIT'); +$running->quit; +$guarded->safe_psql('postgres', "SELECT doltgres_test_move('guarded', 'guard_moved')"); +$holder->query_safe('COMMIT'); +$holder->quit; +my ($guard_root, $guard_folded, $guard_read) = + split / /, $guarded->safe_psql('postgres', "SELECT doltgres_test_base_points('guarded')"); +cmp_ok($guard_read, '<', $guard_folded, + "a move over takes its fold kept above a later writer's lowers the moved base's read point"); +$guarded->safe_psql('postgres', rewritten('guarded', undef, 'false')); +my $guard_rows = "SELECT count(*) || ' ' || md5(string_agg(k || ':' || v, ',' ORDER BY k)) FROM %"; +my $unindexed = 'SET enable_indexscan = off; SET enable_indexonlyscan = off; SET enable_bitmapscan = off;'; +is($guarded->safe_psql('postgres', "$unindexed " . rows('guarded', $guard_rows)), + $guarded->safe_psql('postgres', rows('heap_guarded', $guard_rows)), + "a rewrite of the moved table before any fold carries every row and value as heap's twin holds them"); +is( $guarded->safe_psql('postgres', + q{SELECT count(*) = count(DISTINCT substring(r from '^(\S+) ')) FROM doltgres_test_scan('guarded', false, -1) r}), + 't', 'and gives no tuple id twice'); +$guarded->stop; + done_testing(); diff --git a/src/doltgres/test/t/050_fold.pl b/src/doltgres/test/t/050_fold.pl index 6b7aea45adbe2..c723b55b40eb7 100644 --- a/src/doltgres/test/t/050_fold.pl +++ b/src/doltgres/test/t/050_fold.pl @@ -577,4 +577,87 @@ BEGIN 'the base and a read hold the eleven rows, the loaded ten not applied twice' ); $raced->stop; + +# Bugs row 295: a temporary table's fold and index build are bounded by the +# backend's own snapshots as well as by Postgres' temp horizon, which counts +# no other session's transaction, so with an older transaction open elsewhere +# no base of the table is folded past the snapshot that reads it: a read +# through a new index returns the table's rows, and a fold across an open +# cursor's snapshot leaves the rows the cursor's transaction began after. +my $temped = PostgreSQL::Test::Cluster->new('temped'); +$temped->init; +$temped->start; +$temped->safe_psql('postgres', 'CREATE EXTENSION doltgres_test'); +my $older = $temped->background_psql('postgres'); +$older->query_safe('BEGIN; SELECT pg_current_xact_id()'); +my ($indexed_code, $indexed, $indexed_error) = $temped->psql( + 'postgres', q{ +CREATE TEMP TABLE temp_indexed (a int4); +INSERT INTO temp_indexed VALUES (3), (1), (2); +CREATE INDEX ON temp_indexed (a); +SET enable_seqscan = off; +SET enable_sort = off; +SELECT a FROM temp_indexed ORDER BY a; +}); +is($indexed . $indexed_error, "1\n2\n3", + "a read through a temporary table's new index returns its rows while an older transaction runs elsewhere"); +my ($cursored_code, $cursored, $cursored_error) = $temped->psql( + 'postgres', q{ +CREATE TEMP TABLE temp_cursored (a int4); +INSERT INTO temp_cursored VALUES (3), (1), (2); +CREATE INDEX ON temp_cursored (a); +SET enable_seqscan = off; +SET enable_sort = off; +BEGIN; +DECLARE c CURSOR FOR SELECT a FROM temp_cursored ORDER BY a; +FETCH 1 FROM c; +SELECT doltgres_test_fold('temp_cursored'); +FETCH ALL FROM c; +COMMIT; +SELECT a FROM temp_cursored ORDER BY a; +}); +is($cursored . $cursored_error, "1\nf\n2\n3\n1\n2\n3", + "a fold across an open cursor on a temporary table leaves the rows its snapshot may not read as folded"); +$older->query_safe('COMMIT'); +$older->quit; + +# Bugs row 328: a fold of a keyless table whose ALTER TABLE ... SET STORAGE +# EXTERNAL is pending applies its chain, keeping each entry's copies, those +# of equal rows written before the ALTER and so re-keyed at its commit and +# those written after it, and the table reads every copy back, where the +# re-form refused an entry of more than one copy; the base's sizes count +# every copy's row and heap bytes, as a fold with no chain to apply counts +# the same rows written after the same ALTER. +$temped->safe_psql( + 'postgres', q{ +CREATE TABLE stored_copies (k int4, v text); +INSERT INTO stored_copies SELECT g % 3, repeat(chr(65 + g % 3), 3000) FROM generate_series(1, 6) g; +ALTER TABLE stored_copies ALTER COLUMN v SET STORAGE EXTERNAL; +INSERT INTO stored_copies SELECT g % 3, repeat(chr(65 + g % 3), 3000) FROM generate_series(7, 12) g; +CREATE TABLE stored_after (k int4, v text); +ALTER TABLE stored_after ALTER COLUMN v SET STORAGE EXTERNAL; +SELECT doltgres_test_fold('stored_after'); +INSERT INTO stored_after SELECT g % 3, repeat(chr(65 + g % 3), 3000) FROM generate_series(1, 12) g; +SELECT doltgres_test_fold('stored_after'); +}); +is( $temped->safe_psql( + 'postgres', q{SELECT doltgres_test_fold('stored_copies')}), + 't', + 'a fold of a keyless table with a pending SET STORAGE and equal rows installs a base' +); +is( $temped->safe_psql( + 'postgres', + q{SELECT k, count(*), min(length(v)), max(length(v)) FROM stored_copies GROUP BY k ORDER BY k} + ), + "0|4|3000|3000\n1|4|3000|3000\n2|4|3000|3000", + 'and the table reads every copy of its equal rows back'); +my $base_counts = sub { + return $temped->safe_psql('postgres', + "SELECT doltgres_test_base_sizes('$_[0]')") =~ + s/^(\S+ \S+ \S+).*/$1/r; +}; +is($base_counts->('stored_copies'), $base_counts->('stored_after'), + "and the base's heap bytes, rows, and entries count every copy, as a fold with no chain to apply counts them" +); +$temped->stop; done_testing(); diff --git a/src/doltgres/test/t/053_spill.pl b/src/doltgres/test/t/053_spill.pl index 8585e06cb9b87..bbdc1a4016ecf 100644 --- a/src/doltgres/test/t/053_spill.pl +++ b/src/doltgres/test/t/053_spill.pl @@ -267,11 +267,24 @@ sub spill_files }); ok(spill_files($aborted) > 0, "an aborted transaction's spill files stay until a checkpoint"); +# The versioned collector holds a reader while it sleeps (bugs row 255), which +# keeps every deferred removal waiting, so the case ends it before its +# checkpoints, the postmaster starting it again ten seconds later; this step +# goes when row 255 is fixed. +$node->safe_psql('postgres', + q{SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE backend_type = 'versioned collector'}); +$node->poll_query_until('postgres', + q{SELECT count(*) = 0 FROM pg_stat_activity WHERE backend_type = 'versioned collector'}) + or die 'the versioned collector stayed'; +$node->safe_psql('postgres', 'CHECKPOINT') for 1 .. 2; +isnt(spill_files($aborted), '0', "an aborted transaction's spill files stay through the second checkpoint after it"); +# A fold worker's reader, which the third checkpoint's removal rightly waits +# for, ends with its fold first. +$node->poll_query_until('postgres', + q{SELECT count(*) = 0 FROM pg_stat_activity WHERE backend_type = 'fold worker'}) + or die 'a fold worker stayed'; $node->safe_psql('postgres', 'CHECKPOINT'); -$node->safe_psql('postgres', 'CHECKPOINT'); -is(spill_files($aborted), '0', - "an aborted transaction's spill files go at the second checkpoint after it" -); +is(spill_files($aborted), '0', "and go at the third"); my $crashed = $node->safe_psql( 'postgres', q{ BEGIN; @@ -281,9 +294,9 @@ sub spill_files }); $node->stop('immediate'); $node->start; -$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', 'CHECKPOINT') for 1 .. 2; is(spill_files($crashed), '0', - "a crash between an abort and a checkpoint leaves none of its spill files once a checkpoint after the start ran" + "a crash between an abort and a checkpoint leaves none of its spill files once two checkpoints after the start ran" ); folded('swept'); @@ -355,7 +368,7 @@ sub spilled_in_savepoint # A spill whose write fails between its slot and the end of its removal, in a # savepoint: the versions it took out stay in the memtable too, another -# session's conflict check still finds the key, and the commit's link of the +# session's conflict check still finds the key, and the rollback's link of the # pending file takes the memtable's twins out, so each row reads once. my ($waited, $count) = spilled_in_savepoint('pended', 'doltgres-spill-pended'); @@ -365,7 +378,7 @@ sub spilled_in_savepoint "a spill failed after its slot leaves its key to another writer's conflict check" ); is($count, '3000', - "the commit links a spill that failed after its slot, its memtable twins taken out" + "the rollback links a spill that failed after its slot, its memtable twins taken out" ); folded('pended'); @@ -655,6 +668,73 @@ sub chunk_listed 'a test insert of a source past shared_buffers completes on a node made for streaming'); $pinned->stop; +# Commit 4's case: a statement inserting 300,000 rows at a 4 MB delta_memory +# completes, its spills dropping their rows' tuple ids from the storage's tuple +# id table, whose compaction leaves those slots and their keys' bytes behind, +# and every row reads back. +my $compacted = PostgreSQL::Test::Cluster->new('spill_compacted'); +$compacted->init; +$compacted->append_conf('postgresql.conf', + "delta_memory = 4MB\nfold_threshold = 1GB\nfold_timeout = 3600s\nmax_parallel_workers_per_gather = 0"); +$compacted->start; +$compacted->safe_psql( + 'postgres', q{ +CREATE EXTENSION doltgres_test; +CREATE TABLE many_source (v text); +INSERT INTO many_source SELECT 'r' || i FROM generate_series(1, 300000) i; +CREATE TABLE many (v text); +}); +$compacted->safe_psql('postgres', + q{SELECT count(*) FROM doltgres_test_insert('many', 'many_source', true, 0)}, timeout => 1200); +is($compacted->safe_psql('postgres', q{SELECT count(*) FROM doltgres_test_scan('many', false, -1)}), + '300000', 'a statement inserting 300,000 rows at a 4 MB delta_memory completes and every row reads back'); +$compacted->stop; + +# A standby's replay of an aborted transaction's spills: at a 4 MB delta_memory +# a standby replays a transaction whose rows and free space map takes pass its +# budget, which its redo spills, and then the transaction's abort, which keeps +# the free space map spill marked with the abort's end until a fold takes it; +# the standby reads the table as its primary does after a committed insert, +# after the primary's fold, and after the standby's restart, once it folded at +# its primary's point and its primary's checkpoint gave it a restartpoint, which +# no fold point a storage holds unfolded lets it pass (bugs row 158). +my $aborting = PostgreSQL::Test::Cluster->new('spill_aborting'); +$aborting->init(allows_streaming => 1); +$aborting->append_conf('postgresql.conf', + "delta_memory = 4MB\nfold_threshold = 1GB\nfold_timeout = 3600s\nmax_parallel_workers_per_gather = 0"); +$aborting->start; +$aborting->safe_psql( + 'postgres', q{ +CREATE EXTENSION doltgres_test; +CREATE TABLE aborted_source (v text); +INSERT INTO aborted_source SELECT 'r' || i FROM generate_series(1, 100000) i; +CREATE TABLE aborted_rows (v text); +}); +$aborting->backup('aborting_backup'); +my $aborting_standby = PostgreSQL::Test::Cluster->new('spill_aborting_standby'); +$aborting_standby->init_from_backup($aborting, 'aborting_backup', has_streaming => 1); +$aborting_standby->append_conf('postgresql.conf', "delta_memory = 4MB\nmax_parallel_workers_per_gather = 0"); +$aborting_standby->start; +$aborting->safe_psql('postgres', + q{BEGIN; SELECT count(*) FROM doltgres_test_insert('aborted_rows', 'aborted_source', true, 0); ROLLBACK;}, + timeout => 1200); +$aborting->safe_psql('postgres', + q{SELECT count(*) FROM doltgres_test_insert('aborted_rows', 'aborted_source', true, 0)}, timeout => 1200); +$aborting->wait_for_catchup($aborting_standby); +my $aborted_count = q{SELECT count(*) FROM doltgres_test_scan('aborted_rows', false, -1)}; +is($aborting_standby->safe_psql('postgres', $aborted_count), + '100000', "a standby that replayed an aborted transaction's spills reads only the committed rows"); +$aborting->safe_psql('postgres', q{SELECT doltgres_test_fold('aborted_rows')}); +$aborting->wait_for_catchup($aborting_standby); +is($aborting_standby->safe_psql('postgres', $aborted_count), '100000', "and after the primary's fold"); +$aborting_standby->safe_psql('postgres', q{SELECT doltgres_test_fold('aborted_rows')}); +$aborting->safe_psql('postgres', 'CHECKPOINT'); +$aborting->wait_for_catchup($aborting_standby); +$aborting_standby->restart; +is($aborting_standby->safe_psql('postgres', $aborted_count), '100000', "and after the standby's restart"); +$aborting_standby->stop; +$aborting->stop; + # Bugs row 74 through the real commands: one INSERT ... SELECT whose own # versions pass the default 32 MB of delta_memory completes, its running # transaction spilling them (the delta's drain), and every row reads back. @@ -940,49 +1020,6 @@ sub written $start, "the delta's used bytes are back at their start once both ended and folded"); $small->stop; -# Bugs row 242: an UPDATE ... FROM through a nested loop whose outer scan reads -# a source whose rows its writer spilled, held at its first read of the -# source's spills while a fold of the source installs a base and drops the -# spills, updates every row, as heap's twin's does. -my %joined; -for my $am ('', 'USING doltgres_twin') -{ - my $p = $am eq '' ? '' : 'heap_'; - $node->safe_psql( - 'postgres', qq{ -CREATE TABLE ${p}joined_source (k int4, v text) $am; -INSERT INTO ${p}joined_source SELECT g, repeat('s', 100) FROM generate_series(1, 40000) g; -CREATE TABLE ${p}joined_target (k int4 PRIMARY KEY, v text) $am; -INSERT INTO ${p}joined_target SELECT g, 't' FROM generate_series(1, 40000) g; -}); - my $plan = "SET enable_hashjoin = off; SET enable_mergejoin = off; SET enable_material = off;" - . " SET max_parallel_workers_per_gather = 0;"; - my $update = "UPDATE ${p}joined_target t SET v = s.v || 'u' FROM ${p}joined_source s WHERE t.k = s.k"; - if ($am eq '') - { - my $updater = $node->background_psql('postgres'); - $updater->query_safe( - "$plan SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" - ); - $updater->query_until(qr/started/, "\\echo started\n$update;\n"); - $node->wait_for_event('client backend', 'doltgres-spill-slots-read'); - $node->safe_psql('postgres', "SELECT doltgres_test_fold('joined_source')"); - $node->safe_psql('postgres', - "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" - ); - $updater->query_safe('SELECT 1'); - $updater->quit; - } - else - { - $node->safe_psql('postgres', "$plan $update"); - } - $joined{$p} = $node->safe_psql('postgres', - "SELECT count(*) || ' ' || md5(string_agg(k || v, ',' ORDER BY k)) FROM ${p}joined_target"); -} -is($joined{''}, $joined{heap_}, - "an UPDATE ... FROM whose source scan a fold's drop passes under updates every row as heap's twin's"); - # Bugs row 286: a helper spilling an idle transaction's versions pauses after # taking them out of its memtable and before listing its file, and the # transaction commits; a second transaction's upsert of the same keys, which @@ -1033,4 +1070,125 @@ sub written "a helper's spill listed after the helped transaction's commit leaves a later transaction's spill of the same keys newer" ); +# Bugs row 307: a spill's take-out of versions from the memtable pends the +# spill, which readers read, before the versions leave the memtable. With the +# budget's committed arm paused at doltgres-spill-pended, its spill of a +# keyless table's committed inserts pended and every one still in the +# memtable, a fold of the table waits for the take-out's end and applies each +# insert once. Then, with a running transaction's inserts spilled so that +# readers list the table's spills, a count paused at +# doltgres-spill-slots-read, having read the first key's chain, across a +# whole take-out of the table's newer committed inserts reads each row once. +my $taken = PostgreSQL::Test::Cluster->new('taken'); +$taken->init; +$taken->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\nautovacuum = off"); +$taken->start; +$taken->safe_psql('postgres', + 'CREATE EXTENSION injection_points; CREATE EXTENSION doltgres_test; CREATE TABLE taken_once (k int4, v text)' +); +$taken->safe_psql('postgres', + "INSERT INTO taken_once SELECT n, repeat('t', 100) FROM generate_series(1, 1000) n" +); +my $spiller = $taken->background_psql('postgres'); +$spiller->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-pended', 'wait')" +); +$spiller->query_until(qr/spilling/, + "\\echo spilling\nSELECT doltgres_test_delta(61, 0, 0);\n"); +$taken->wait_for_event('client backend', 'doltgres-spill-pended'); +my $folder = $taken->background_psql('postgres'); +my $folder_pid = $folder->query_safe('SELECT pg_backend_pid()'); +$folder->query_until(qr/folding/, + "\\echo folding\nSELECT doltgres_test_fold('taken_once');\n"); +$taken->poll_query_until('postgres', + "SELECT count(*) = 1 FROM pg_stat_activity WHERE pid = $folder_pid AND (state = 'idle' OR wait_event = 'Extension')" +) or die 'the fold neither ended nor waited'; +$taken->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-pended'); SELECT injection_points_wakeup('doltgres-spill-pended')" +); +$spiller->query_safe('SELECT 1'); +$folder->query_safe('SELECT 1'); +is( $taken->safe_psql( + 'postgres', + q{SELECT count(*) || ' ' || count(DISTINCT k) || ' ' || (split_part(doltgres_test_count('taken_once'), ' ', 2) = split_part(doltgres_test_count('taken_once'), ' ', 6)) FROM taken_once} + ), + '1000 1000 true', + 'a fold that read a keyless table while a take-out of its inserts was pended applies each insert once' +); +$taken->safe_psql('postgres', + "INSERT INTO taken_once SELECT n, repeat('t', 100) FROM generate_series(1001, 2000) n" +); +my $running = $taken->background_psql('postgres'); +$running->query_safe('BEGIN'); +$running->query_safe( + "INSERT INTO taken_once SELECT n, repeat('t', 100) FROM generate_series(2001, 2100) n" +); +is($taken->safe_psql('postgres', 'SELECT doltgres_test_delta(61, 0, 0)'), + '1', "a helper spills the running transaction's inserts, so the table's spills are listed"); +my $counter = $taken->background_psql('postgres'); +$counter->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" +); +$counter->query_until(qr/counting/, + "\\echo counting\nSELECT count(*) FROM taken_once;\n"); +$taken->wait_for_event('client backend', 'doltgres-spill-slots-read'); +is($taken->safe_psql('postgres', 'SELECT doltgres_test_delta(61, 0, 0)'), + '1', "the committed arm takes the newer inserts out while the count waits"); +$taken->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" +); +like($counter->query_safe('SELECT 1'), + qr/^2000$/m, + 'a count that read a key before a whole take-out of it and listed the spills after reads it once'); +$running->query_safe('ROLLBACK'); +$running->quit; +$spiller->quit; +$folder->quit; +$counter->quit; + +# A take-out that a top-level error stops between its spill's pend and its +# versions' removal ends at the transaction's abort, and one whose backend is +# terminated there ends at its exit, so another session's read of the table +# finishes and reads each row once. +$taken->safe_psql('postgres', "SELECT doltgres_test_fold('taken_once')"); +$taken->safe_psql('postgres', + "CREATE TABLE taken_ended (k int4, v text); INSERT INTO taken_ended SELECT n, repeat('t', 100) FROM generate_series(1, 1000) n" +); +my ($code, $out, $err) = $taken->psql('postgres', + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-pended', 'error'); SELECT doltgres_test_delta(61, 0, 0)" +); +like($err, qr/error triggered for injection point doltgres-spill-pended/, + "the committed arm's take-out fails between its pend and its removal"); +is( $taken->safe_psql( + 'postgres', + "SET statement_timeout = '60s'; SELECT count(*) FROM taken_ended"), + '1000', + 'a read after a take-out a top-level error stopped finishes and reads each row once' +); +$taken->safe_psql('postgres', "SELECT doltgres_test_fold('taken_ended')"); +$taken->safe_psql('postgres', + "CREATE TABLE taken_exited (k int4, v text); INSERT INTO taken_exited SELECT n, repeat('t', 100) FROM generate_series(1, 1000) n" +); +my $exiting = $taken->background_psql('postgres', on_error_stop => 0); +my $exiting_pid = $exiting->query_safe('SELECT pg_backend_pid()'); +$exiting->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-pended', 'wait')" +); +$exiting->query_until(qr/exiting/, + "\\echo exiting\nSELECT doltgres_test_delta(61, 0, 0);\n"); +$taken->wait_for_event('client backend', 'doltgres-spill-pended'); +$taken->safe_psql('postgres', "SELECT pg_terminate_backend($exiting_pid)"); +$taken->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_stat_activity WHERE pid = $exiting_pid") + or die 'the terminated backend never exited'; +$exiting->quit; +is( $taken->safe_psql( + 'postgres', + "SET statement_timeout = '60s'; SELECT count(*) FROM taken_exited"), + '1000', + 'a read after a take-out whose backend was terminated finishes and reads each row once' +); +$taken->stop; + done_testing(); diff --git a/src/doltgres/test/t/057_identity.pl b/src/doltgres/test/t/057_identity.pl index dc82ccaed41f6..1d34a082c038a 100644 --- a/src/doltgres/test/t/057_identity.pl +++ b/src/doltgres/test/t/057_identity.pl @@ -387,4 +387,41 @@ sub forked "a rewrite moves the table's TOAST pair to its new TOAST relation"); $node->stop; + +# Bugs row 338: a DROP TABLE of a table whose row type grew by 200 columns +# removes its 404 identity entries in groups the budget can give, at 8MB and +# at 1MB, where one prepare asked more pages than the whole budget holds and +# waited forever. +foreach my $memory ('8MB', '1MB') +{ + my $grown = PostgreSQL::Test::Cluster->new("grown_$memory"); + $grown->init; + $grown->append_conf('postgresql.conf', "delta_memory = $memory"); + $grown->start; + $grown->safe_psql( + 'postgres', q{ +CREATE TABLE compositetable_type (a int); +CREATE TABLE compositetable_holder (id int, c compositetable_type); +INSERT INTO compositetable_holder VALUES (1, ROW(1)::compositetable_type); +DO $$ +BEGIN + FOR i IN 1..200 LOOP + EXECUTE format('ALTER TABLE compositetable_type ADD COLUMN x%s bigint', i); + END LOOP; +END $$; +DROP TABLE compositetable_holder; +}); + my ($code, $out, $err) = $grown->psql( + 'postgres', q{ +SET statement_timeout = '120s'; +DROP TABLE compositetable_type; +SELECT count(*) FROM pg_class WHERE relname = 'compositetable_type'; +}); + is( $err . $out, + '0', + "a DROP TABLE of a table grown by 200 columns finishes at delta_memory $memory" + ); + $grown->stop; +} + done_testing(); diff --git a/src/doltgres/test/t/062_fold_trigger.pl b/src/doltgres/test/t/062_fold_trigger.pl index 53ff367d1e2b8..c4766d864dcb1 100644 --- a/src/doltgres/test/t/062_fold_trigger.pl +++ b/src/doltgres/test/t/062_fold_trigger.pl @@ -767,13 +767,21 @@ BEGIN # 062's budget case (bugs rows 111 and 160) through the real commands: with # delta_memory at 1 MB and neither of the fold's thresholds met, committed # writes far past it finish, a writer refused at the budget asking for folds -# whose drops give the delta back and wake it. +# whose drops give the delta back and wake it. A count right after the last +# commit, held after it read the table's spill slots while a fold installs a +# base that applies the spilled rows and drops them from the delta, moves to +# that base at its next read and counts every row (bugs row 241); the fold +# launcher is held from before the count, so no fold but the case's own +# installs a base. my $budget = PostgreSQL::Test::Cluster->new('budget'); $budget->init; $budget->append_conf('postgresql.conf', - "delta_memory = 1MB\nfold_threshold = '1GB'\nfold_timeout = 3600s"); + "delta_memory = 1MB\nfold_threshold = '1GB'\nfold_timeout = 3600s\nautovacuum = off" +); $budget->start; -$budget->safe_psql('postgres', 'CREATE TABLE committed (k int4, v text)'); +$budget->safe_psql('postgres', + 'CREATE EXTENSION injection_points; CREATE EXTENSION doltgres_test; CREATE TABLE committed (k int4, v text)' +); my $refused = 0; for my $batch (0 .. 49) { @@ -785,8 +793,185 @@ BEGIN } is($refused, 0, "fifty committed writes past a 1 MB delta_memory finish with neither of the fold's thresholds met"); +$budget->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-fold-launch', 'wait')"); +$budget->poll_query_until('postgres', + q{SELECT count(*) = 1 FROM pg_stat_activity WHERE wait_event = 'doltgres-fold-launch'} +) or die 'the fold launcher never waited'; +$budget->poll_query_until('postgres', + q{SELECT count(*) = 0 FROM pg_stat_activity WHERE backend_type = 'fold worker'} +) or die 'a fold worker kept running'; +my $counter = $budget->background_psql('postgres'); +$counter->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" +); +$counter->query_until(qr/held/, + "\\echo held\nSELECT count(*) FROM committed;\n"); +$budget->wait_for_event('client backend', 'doltgres-spill-slots-read'); +is( $budget->safe_psql('postgres', "SELECT doltgres_test_fold('committed')"), + 't', + 'a fold under the held count installs a base'); +$budget->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-fold-launch'); SELECT injection_points_wakeup('doltgres-fold-launch')" +); +$budget->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" +); +my $counted = $counter->query_safe('SELECT 1'); +$counter->quit; +like($counted, qr/^50000$/m, + 'a count right after the last commit, held across a fold that applied its spilled rows, counts every row' +); is($budget->safe_psql('postgres', 'SELECT count(*) FROM committed'), '50000', 'and every row reads back'); $budget->stop; +# Bugs row 306: a fold worker held inside its fold, its snapshot taken, keeps +# neither CREATE INDEX CONCURRENTLY nor REINDEX TABLE CONCURRENTLY of other +# tables waiting for it, as a VACUUM doesn't. +my $built = PostgreSQL::Test::Cluster->new('built'); +$built->init; +$built->append_conf('postgresql.conf', + "fold_threshold = 16kB\nfold_timeout = 3600s\nautovacuum = off"); +$built->start; +$built->safe_psql( + 'postgres', q{ +CREATE EXTENSION injection_points; +CREATE EXTENSION doltgres_test; +CREATE TABLE indexed (k int4); +INSERT INTO indexed SELECT generate_series(1, 100); +CREATE TABLE reindexed (k int4); +INSERT INTO reindexed SELECT generate_series(1, 100); +CREATE INDEX reindexed_k ON reindexed (k); +CREATE TABLE held (k int4, v text); +SELECT injection_points_attach('doltgres-fold-relation', 'wait'); +}); +$built->safe_psql('postgres', + "SELECT doltgres_test_write('held', true, 'INSERT INTO held SELECT n, repeat(''h'', 200) FROM generate_series(1, 200) n')" +); +$built->wait_for_event('fold worker', 'doltgres-fold-relation'); +is( $built->safe_psql( + 'postgres', + q{SELECT count(*) FROM pg_stat_activity WHERE backend_type = 'fold worker' AND wait_event = 'doltgres-fold-relation' AND backend_xmin IS NOT NULL} + ), + '1', + 'the held fold worker has its snapshot'); +my ($built_ret, $built_out, $built_err) = $built->psql('postgres', + "SET lock_timeout = '30s'; CREATE INDEX CONCURRENTLY indexed_k ON indexed (k)" +); +is($built_ret, 0, + "CREATE INDEX CONCURRENTLY doesn't wait for a fold worker held inside its fold" +); +($built_ret, $built_out, $built_err) = $built->psql('postgres', + "SET lock_timeout = '30s'; REINDEX TABLE CONCURRENTLY reindexed"); +is($built_ret, 0, + "REINDEX TABLE CONCURRENTLY doesn't wait for a fold worker held inside its fold" +); +$built->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-fold-relation'); SELECT injection_points_wakeup('doltgres-fold-relation')" +); +ok( $built->poll_query_until( + 'postgres', + "SELECT split_part(doltgres_test_base_sizes('held'), ' ', 2) = '200'" + ), + 'the released fold installs its base'); +is( $built->safe_psql( + 'postgres', + 'SET enable_seqscan = off; SELECT count(*) FROM indexed WHERE k <= 50; SELECT count(*) FROM reindexed WHERE k <= 50' + ), + "50\n50", + 'and the built and the rebuilt index read their tables'); + +# A CREATE INDEX CONCURRENTLY of the very table a held fold worker folds, its +# snapshot older than the deletes and inserts made before the build, doesn't +# wait for it, and once the fold installs its base the new index gives every +# row a sequential scan gives. +$built->safe_psql( + 'postgres', q{ +CREATE TABLE own (k int4, v text); +INSERT INTO own SELECT n, 'o' FROM generate_series(1, 100) n; +SELECT injection_points_attach('doltgres-fold-relation', 'wait'); +}); +$built->safe_psql('postgres', + "SELECT doltgres_test_write('own', true, 'INSERT INTO own SELECT n, repeat(''o'', 200) FROM generate_series(101, 300) n')" +); +$built->wait_for_event('fold worker', 'doltgres-fold-relation'); +$built->safe_psql('postgres', + q{DELETE FROM own WHERE k <= 20; INSERT INTO own SELECT n, 'n' FROM generate_series(301, 350) n} +); +($built_ret, $built_out, $built_err) = $built->psql('postgres', + "SET lock_timeout = '30s'; CREATE INDEX CONCURRENTLY own_k ON own (k)"); +is($built_ret, 0, + "CREATE INDEX CONCURRENTLY of a table a held fold worker folds doesn't wait for it" +); +$built->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-fold-relation'); SELECT injection_points_wakeup('doltgres-fold-relation')" +); +ok( $built->poll_query_until( + 'postgres', + q{SELECT count(*) = 0 FROM pg_stat_activity WHERE backend_type = 'fold worker'} + ), + "the table's released fold ends"); +my $counted_rows = q{SELECT count(*) || ' ' || sum(k) FROM own WHERE k > 0}; +my $by_index = + "SET enable_seqscan = off; SET enable_bitmapscan = off; $counted_rows"; +my $by_scan = + "SET enable_indexscan = off; SET enable_bitmapscan = off; SET enable_indexonlyscan = off; $counted_rows"; +is($built->safe_psql('postgres', $by_index), + '330 61215', 'and the new index gives every row'); +is($built->safe_psql('postgres', $by_scan), + '330 61215', 'as a sequential scan does'); +$built->safe_psql('postgres', + "SELECT doltgres_test_fold('own'); SELECT doltgres_test_index_fold('own_k')" +); +is($built->safe_psql('postgres', $by_index), + '330 61215', + 'and still does once the table and the index fold every row'); +$built->stop; + +# A replayed free's take-out passes over a later take of its tuple id (bugs +# row 320): with a standby's launcher held, the primary folds three rounds of +# a comparator-keyed table, each deleting a row and freeing its tuple id at a +# VACUUM, the next round's insert taking that tuple id again on another key, +# and a fourth round's take of it stays live. Once the launcher runs, the +# standby folds each point and does each free's deferred take-outs past it, +# which meet the later takes; every root matches its primary's at each fold, +# and after a last VACUUM every row reads back as on the primary. +$node->safe_psql('postgres', + "ALTER SYSTEM SET fold_threshold = '1GB'; ALTER SYSTEM SET fold_timeout = '3600s'; SELECT pg_reload_conf()" +); +$node->backup('retaken'); +my $retaken = PostgreSQL::Test::Cluster->new('retaken'); +$retaken->init_from_backup($node, 'retaken', has_streaming => 1); +$retaken->start; +ok($held_launcher->($retaken), "the retaken standby's launcher is held"); +$node->safe_psql('postgres', + 'CREATE TABLE retaken_keys (k ranged PRIMARY KEY, v text)'); +$folds->('retaken_keys', 3); +$node->safe_psql('postgres', + "SELECT doltgres_test_write('retaken_keys', true, 'INSERT INTO retaken_keys SELECT ranged(n, n + 1), ''r'' FROM generate_series(31, 40) n')" +); +$node->safe_psql('postgres', "SELECT doltgres_test_fold('retaken_keys')"); +$node->safe_psql('postgres', + "SELECT doltgres_test_vacuum('retaken_keys', 'auto')"); +$node->wait_for_replay_catchup($retaken); +$offset = -s $retaken->logfile; +$run_launcher->($retaken); +ok( $same_base->($retaken, 'retaken_keys'), + "a lagging standby whose deferred take-outs meet later takes of their tuple ids folds each point to its primary's roots" +); +ok( !$retaken->log_contains(qr/does not match its primary's/, $offset), + "the retaken standby's rows, tuple id, free space, and pending tuple id roots match each record's" +); +$node->safe_psql('postgres', + "SELECT doltgres_test_vacuum('retaken_keys', 'auto')"); +$node->wait_for_replay_catchup($retaken); +my $retaken_rows = + q{SELECT doltgres_test_standing('retaken_keys', 'SELECT string_agg(lower(k)::text, '' '' ORDER BY lower(k)) FROM retaken_keys')}; +is( $retaken->safe_psql('postgres', $retaken_rows), + $node->safe_psql('postgres', $retaken_rows), + "after a last VACUUM every row, the live retake's included, reads back on the standby as on its primary" +); +$retaken->stop; + done_testing(); diff --git a/src/doltgres/test/t/062_size_sites.pl b/src/doltgres/test/t/062_size_sites.pl index e5968b0afcdf5..c306916ea1880 100644 --- a/src/doltgres/test/t/062_size_sites.pl +++ b/src/doltgres/test/t/062_size_sites.pl @@ -55,9 +55,12 @@ ); is( $node->safe_psql( 'postgres', q{ -SELECT pg_relation_size('sized', 'fsm'), pg_relation_size('sized', 'vm'), pg_relation_size('sized', 'init')}), - '0|0|0', - 'a fork table\'s other forks have no size'); +SELECT pg_relation_size('sized', 'fsm') + = coalesce((pg_stat_file(pg_relation_filepath('sized') || '_fsm', true)).size, 0), + pg_relation_size('sized', 'vm'), pg_relation_size('sized', 'init')}), + 't|0|0', + 'a fork table\'s free space map fork is its real pages, and its other forks have no size' +); is( $node->safe_psql( 'postgres', q{ SELECT pg_relation_size('sized_k') % 8192 = 0 AND pg_relation_size('sized_k') >= 16384, @@ -420,5 +423,223 @@ sub sized is(sized($standby, 'stood'), $primary_sizes, 'a promoted standby\'s sizes equal its primary\'s'); $standby->stop; +# Bugs rows 240 and 316: a commit's totals, held where the commit reads its +# storage's base before a fold installs a base holding another transaction's +# rows and drops their versions from the delta, move by what the commit's +# rows change, so a DELETE of those rows leaves a keyless table's totals, and +# an UPDATE that keeps their values leaves a TOAST relation's, equal to a fresh +# count; and a reader's count of totals a rewrite left unknown, held where +# the count reads the base before such a fold, counts them again to a fresh +# count. +sub launcher_held +{ + $node->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-fold-launch', 'wait')"); + $node->poll_query_until('postgres', + q{SELECT count(*) = 1 FROM pg_stat_activity WHERE wait_event = 'doltgres-fold-launch'} + ) or die 'the fold launcher never waited'; + $node->poll_query_until('postgres', + q{SELECT count(*) = 0 FROM pg_stat_activity WHERE backend_type = 'fold worker'} + ) or die 'a fold worker kept running'; +} + +sub launcher_freed +{ + $node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-fold-launch'); SELECT injection_points_wakeup('doltgres-fold-launch')" + ); +} + +sub held_fold +{ + my ($table, $point, $before, $held) = @_; + launcher_held(); + $node->safe_psql('postgres', $before); + my $session = $node->background_psql('postgres', on_error_stop => 0); + $session->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('$point', 'wait')" + ); + $session->query_until(qr/held/, "\\echo held\n$held;\n"); + $node->wait_for_event('client backend', $point); + is($node->safe_psql('postgres', "SELECT doltgres_test_fold('$table')"), + 't', "a fold under a backend held at $point installs a base"); + launcher_freed(); + $node->safe_psql('postgres', + "SELECT injection_points_detach('$point'); SELECT injection_points_wakeup('$point')" + ); + $session->query('SELECT 1'); + $session->quit; +} +held_fold( + 'moved_rows', + 'doltgres-commit-based', + q{CREATE TABLE moved_rows (k int4, v text); +INSERT INTO moved_rows SELECT g, 'v' || g FROM generate_series(1, 100) g}, + 'BEGIN; DELETE FROM moved_rows WHERE k <= 50; COMMIT'); +is(totals('moved_rows'), count('moved_rows'), + 'a commit held across a fold that applied the rows it deletes moves a keyless table\'s totals to a fresh count' +); +$node->safe_psql( + 'postgres', q{ +CREATE TABLE moved_values (k int4, v text); +ALTER TABLE moved_values ALTER COLUMN v SET STORAGE EXTERNAL; +}); +my $moved_toast = $node->safe_psql('postgres', + q{SELECT reltoastrelid::regclass FROM pg_class WHERE relname = 'moved_values'} +); +held_fold( + $moved_toast, + 'doltgres-commit-valued', + q{INSERT INTO moved_values SELECT g, repeat(chr(64 + g), 3000) FROM generate_series(1, 20) g}, + 'BEGIN; UPDATE moved_values SET k = k + 100 WHERE k <= 10; COMMIT'); +is(totals($moved_toast), count($moved_toast), + 'a commit held across a fold that applied the values its update keeps moves a TOAST relation\'s totals to a fresh count' +); +held_fold( + 'recounted', + 'doltgres-recount-based', + q{CREATE TABLE recounted (k int4); +INSERT INTO recounted SELECT g FROM generate_series(1, 10) g; +ALTER TABLE recounted ALTER COLUMN k TYPE int8; +INSERT INTO recounted SELECT g FROM generate_series(11, 20) g}, + q{SELECT pg_relation_size('recounted')}); +is(totals('recounted'), count('recounted'), + 'a count held across a fold counts the totals again to a fresh count'); + +# Bugs row 316: a DELETE after a count that a fold's install met, as merge's +# DELETE and ANALYZE met one, takes its rows from totals the count left right: +# after a crash's restart leaves a folded table's totals unknown, a reader's +# count held across a fold of rows inserted since is followed by a DELETE of +# more rows than the older base holds, and the table's size still reads and +# its totals equal a fresh count, where totals counted over the older base +# wrapped past zero. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE restarted_count (k int4); +INSERT INTO restarted_count SELECT g FROM generate_series(1, 10) g; +SELECT doltgres_test_fold('restarted_count'); +}); +$node->stop('immediate'); +$node->start; +held_fold( + 'restarted_count', + 'doltgres-recount-based', + q{INSERT INTO restarted_count SELECT g FROM generate_series(11, 20) g}, + q{SELECT pg_relation_size('restarted_count')}); +$node->safe_psql('postgres', 'DELETE FROM restarted_count WHERE k <= 15'); +$node->psql( + 'postgres', q{SELECT pg_relation_size('restarted_count')}, + stdout => \my $restarted_size, + stderr => \my $restarted_error); +is($restarted_error, '', + 'a DELETE after a count held across a fold leaves the table\'s size readable' +); +is(totals('restarted_count'), count('restarted_count'), + 'a DELETE after a count held across a fold moves the totals to a fresh count' +); + +# Bugs row 316's carriers: a rewrite that carries versions into a new storage, +# which an older snapshot makes it do by holding the fold horizon below their +# rows, leaves that storage's totals unknown, since its base's sizes count +# only the rows folded into it, so after a DELETE and a VACUUM the relation's +# size reads and equals a twin's that no snapshot held, and a table's totals +# equal a fresh count, where totals of the base alone wrapped past zero: +# merge's primary key added to a keyless table, an index's build, and a +# table's move to another tablespace. +my $spaced = $node->basedir . '/carried_space'; +mkdir $spaced or die "mkdir $spaced: $!"; +$node->safe_psql('postgres', + "CREATE TABLESPACE carried_space LOCATION '$spaced'"); +my %carriers = ( + rekeyed => [ '%T', 'ALTER TABLE %T ADD PRIMARY KEY (tid)' ], + indexed => [ '%T_tid', 'CREATE INDEX %T_tid ON %T (tid)' ], + moved => [ '%T', 'ALTER TABLE %T SET TABLESPACE carried_space' ],); +my %carrier_named = ( + rekeyed => 'a key change', + indexed => 'an index build', + moved => 'a move to another tablespace'); +for my $carrier ('rekeyed', 'indexed', 'moved') +{ + my $named = $carrier_named{$carrier}; + my %size; + for my $table ($carrier, "${carrier}_twin") + { + my ($sized, $carry) = @{ $carriers{$carrier} }; + s/%T/$table/g for ($sized, $carry); + my $horizon = $node->background_psql('postgres'); + $horizon->query_safe( + 'BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT 1') + if $table eq $carrier; + $node->safe_psql('postgres', + "CREATE TABLE $table (tid int4, balance int4); INSERT INTO $table SELECT g, 0 FROM generate_series(1, 20) g; $carry" + ); + $horizon->query_safe('COMMIT') if $table eq $carrier; + $horizon->quit; + $node->safe_psql('postgres', "DELETE FROM $table WHERE tid > 5"); + $node->safe_psql('postgres', "VACUUM $table"); + $node->psql( + 'postgres', "SELECT pg_relation_size('$sized')", + stdout => \$size{$table}, + stderr => \my $carried_error); + is($carried_error, '', + "a DELETE and VACUUM after $named leave the size readable") + if $table eq $carrier; + is(totals($sized), count($sized), + "a DELETE and VACUUM after $named that carried its rows move the totals to a fresh count" + ) if $table eq $carrier && $carrier ne 'indexed'; + } + is($size{$carrier}, $size{"${carrier}_twin"}, + "after $named that carried its rows, the size is a twin's that no snapshot held" + ); +} + +# Bugs row 323: a commit's moves count only the versions its storage's base +# didn't apply. After a fold's install is made durable and a crash's restart +# replays the versions that fold applied, a reader's count makes the totals +# known, and an UPDATE's commit moves a keyless table's totals to a fresh +# count, where counting the replayed versions again added the folded rows +# twice. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE replayed (k int4, v text); +INSERT INTO replayed SELECT g % 3, 'v' FROM generate_series(1, 10) g; +SELECT doltgres_test_fold('replayed'); +CREATE TABLE replayed_flushed (k int4); +}); +$node->stop('immediate'); +$node->start; +launcher_held(); +$node->safe_psql('postgres', q{SELECT pg_relation_size('replayed')}); +$node->safe_psql('postgres', + q{BEGIN; UPDATE replayed SET v = 's' WHERE k = 0; COMMIT}); +launcher_freed(); +is(totals('replayed'), count('replayed'), + 'a commit after a restart replayed what a fold applied moves a keyless table\'s totals to a fresh count' +); + +# Bugs row 312: every block of a fork table is all-visible and all-frozen +# (Differences row 10), so CREATE INDEX, ANALYZE, and REINDEX, which recount +# through visibilitymap_count, write relallvisible and relallfrozen equal to +# relpages, as the fork's VACUUM does, on a table no VACUUM has touched. +my $all_visible = q{ +SELECT relpages > 0, relallvisible = relpages, relallfrozen = relpages FROM pg_class WHERE relname = '%T'}; +$node->safe_psql('postgres', + 'CREATE TABLE counted_indexed (id int4); INSERT INTO counted_indexed SELECT g FROM generate_series(1, 3000) g; CREATE INDEX ON counted_indexed (id)' +); +is($node->safe_psql('postgres', $all_visible =~ s/%T/counted_indexed/r), + 't|t|t', 'CREATE INDEX counts a fork table\'s every block all-visible'); +$node->safe_psql('postgres', + 'CREATE TABLE counted_analyzed (id int4); INSERT INTO counted_analyzed SELECT g FROM generate_series(1, 3000) g; ANALYZE counted_analyzed' +); +is($node->safe_psql('postgres', $all_visible =~ s/%T/counted_analyzed/r), + 't|t|t', 'ANALYZE counts a fork table\'s every block all-visible'); +$node->safe_psql('postgres', + 'INSERT INTO counted_indexed SELECT g FROM generate_series(3001, 9000) g; REINDEX TABLE counted_indexed' +); +is( $node->safe_psql('postgres', $all_visible =~ s/%T/counted_indexed/r), + 't|t|t', + 'REINDEX counts a fork table\'s every block all-visible after more writes' +); + $node->stop; done_testing(); diff --git a/src/doltgres/test/t/065_map_counts.pl b/src/doltgres/test/t/065_map_counts.pl index 7792efff1136a..1fe7e3f5e965a 100644 --- a/src/doltgres/test/t/065_map_counts.pl +++ b/src/doltgres/test/t/065_map_counts.pl @@ -37,8 +37,11 @@ # entry, so the entry stays while the version does. A drop as the status # table's move makes one, of an ended entry nothing counts, then can't # take the entry of a writer whose values or takes are still in a delta: a -# TOAST relation's fold after its table's fold, and a fold that meets a take -# whose row ended before an earlier fold, both read the writer's commit. +# TOAST relation's fold after its table's fold reads the writer's commit. A +# take whose row ended before a fold leaves the delta with its block's run, +# its tuple id pending in the base with its ender's full xid and its value +# ids, so its writer's entry counts nothing, and a later fold after the +# entry's drop asks nothing of it (bugs row 347). sub dropped_if_uncounted { my ($xid) = @_; @@ -71,12 +74,14 @@ sub dropped_if_uncounted q{BEGIN; SELECT doltgres_test_insert('spent', 'source_spent', false, 0); SELECT txid_current(); COMMIT}))[-1]; $node->safe_psql('postgres', q{SELECT doltgres_test_write('spent', true, 'DELETE FROM spent WHERE a = 1'); SELECT doltgres_test_fold('spent')}); -ok(dropped_if_uncounted($taker) > 0, - 'a writer\'s entry still counts its take of a row ended before the fold'); +is( $node->safe_psql('postgres', q{SELECT doltgres_test_mapped('spent')}) . ' ' + . dropped_if_uncounted($taker), + 'f -1', + 'a take of a row ended before the fold leaves the delta, and its writer\'s entry counts nothing'); $node->safe_psql('postgres', q{SELECT doltgres_test_insert('spent', 'source_spent', false, 0)}); (undef, $folded) = $node->psql('postgres', q{SELECT doltgres_test_fold('spent')}); -is($folded, 't', 'a later fold that meets the take reads its writer\'s commit'); +is($folded, 't', 'a later fold after the writer\'s entry is dropped completes'); # An abort and a rollback to a savepoint give back the counts of the versions @@ -136,5 +141,78 @@ sub dropped_if_uncounted is($node->safe_psql('postgres', "SELECT doltgres_test_status_table(1, $locker, 0)"), '0', 'a row lock rolled back to a savepoint leaves its committed transaction\'s entry counting nothing'); + +# Every free space map version a write pushes counts at its writer's entry, +# and a fold gives back only what was counted (bugs row 305). An UPDATE of 40 +# rows on full pages flags a block for each row it moves off one, 40 flags; +# an INSERT of 20 rows into another table by the same transaction pushes 20 +# rows and their 20 takes, 40 versions, as many as the flags. The first +# table's fold gives back the UPDATE's versions, flags included, and leaves +# the entry counting the INSERT's 40, so a drop of an ended entry that counts +# nothing passes it by, and the second table's fold reads the writer's +# commit. With the fold launcher held and no fold worker it started before +# its hold left running, no fold but the case's own folds or drops a version +# while the case counts, and each fold waits until no other process holds an +# xmin or an xid at or below the writer, such as an autovacuum worker's, +# which would keep the fold's horizon below it. +sub horizon_past +{ + my ($xid) = @_; + $node->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND (age(backend_xmin) >= age('$xid'::xid) OR age(backend_xid) >= age('$xid'::xid))" + ) or die "a process holds the horizon at or below $xid"; + return; +} +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); +$node->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-fold-launch', 'wait')"); +$node->poll_query_until('postgres', + "SELECT count(*) = 1 FROM pg_stat_activity WHERE wait_event = 'doltgres-fold-launch'" +) or die 'the fold launcher is not held'; +$node->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_stat_activity WHERE backend_type = 'fold worker'" +) or die 'a fold worker still runs'; +my $filler = ( + split /\n/, + $node->safe_psql( + 'postgres', q{ +CREATE TABLE flagged (id int4 PRIMARY KEY, v text); +BEGIN; +INSERT INTO flagged SELECT g, repeat('x', 900) FROM generate_series(1, 400) g; +SELECT txid_current(); +COMMIT; +CREATE TABLE inserted (id int4, v text); +}))[-1]; +horizon_past($filler); +$node->safe_psql('postgres', q{SELECT doltgres_test_fold('flagged')}); +my $flagger = ( + split /\n/, + $node->safe_psql( + 'postgres', q{ +BEGIN; +UPDATE flagged SET v = repeat('y', 900) WHERE id <= 40; +INSERT INTO inserted SELECT g, 'b' FROM generate_series(1, 20) g; +SELECT txid_current(); +COMMIT; +}))[-1]; +horizon_past($flagger); +$node->safe_psql('postgres', q{SELECT doltgres_test_fold('flagged')}); +is(dropped_if_uncounted($flagger), '40', + 'a writer\'s entry counts its other table\'s versions after its first table\'s fold gives back its flags' +); +(undef, $folded) = + $node->psql('postgres', q{SELECT doltgres_test_fold('inserted')}); +is($folded, 't', 'the other table\'s fold reads its writer\'s commit'); +is( $node->safe_psql( + 'postgres', "SELECT doltgres_test_status_table(1, $flagger, 0)"), + '0', + 'the writer\'s entry counts nothing once both tables fold'); +is( $node->safe_psql( + 'postgres', + q{SELECT (SELECT count(*) FROM flagged WHERE v LIKE 'y%'), (SELECT count(*) FROM inserted)} + ), + '40|20', + 'both tables read the transaction\'s rows'); + $node->stop; done_testing(); diff --git a/src/doltgres/test/t/071_spill_reads.pl b/src/doltgres/test/t/071_spill_reads.pl new file mode 100644 index 0000000000000..42ed2224f6507 --- /dev/null +++ b/src/doltgres/test/t/071_spill_reads.pl @@ -0,0 +1,487 @@ +# Copyright 2026 Dolthub, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Reads held across a fold that applies a committed transaction's spilled +# versions and drops them from the delta (bugs rows 240, 241, and 242): each +# read, held after it read its storage's spill slots, moves to the fold's base +# at its next read and reads every row once. The node's delta_memory is 4 MB, +# so a batch of a few thousand rows spills and stays inside the budget. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('spill_reads'); +$node->init; +$node->append_conf('postgresql.conf', + "delta_memory = 4MB\nfold_threshold = 1GB\nfold_timeout = 3600s\nmax_parallel_workers_per_gather = 0\nautovacuum = off" +); +$node->start; +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} +$node->safe_psql( + 'postgres', q{ +CREATE EXTENSION injection_points; +CREATE EXTENSION doltgres_test; +}); + +# launcher_held holds the fold launcher at its injection point and waits for +# its fold workers to end, so no fold but a case's own installs a base under +# the case's held read before that fold; launcher_freed lets the launcher go +# on. The node runs without autovacuum, whose workers' snapshots would hold +# the fold horizon below a case's batch. +sub launcher_held +{ + $node->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-fold-launch', 'wait')"); + $node->poll_query_until('postgres', + q{SELECT count(*) = 1 FROM pg_stat_activity WHERE wait_event = 'doltgres-fold-launch'} + ) or die 'the fold launcher never waited'; + $node->poll_query_until('postgres', + q{SELECT count(*) = 0 FROM pg_stat_activity WHERE backend_type = 'fold worker'} + ) or die 'a fold worker kept running'; +} + +sub launcher_freed +{ + $node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-fold-launch'); SELECT injection_points_wakeup('doltgres-fold-launch')" + ); +} + +# Bugs row 242: an UPDATE ... FROM through a nested loop whose outer scan reads +# a source whose rows its writer spilled, held at its first read of the +# source's spills while a fold of the source installs a base and drops the +# spills, updates every row, as heap's twin's does. +my %joined; +for my $am ('', 'USING doltgres_twin') +{ + my $p = $am eq '' ? '' : 'heap_'; + launcher_held() if $am eq ''; + $node->safe_psql( + 'postgres', qq{ +CREATE TABLE ${p}joined_source (k int4, v text) $am; +INSERT INTO ${p}joined_source SELECT g, repeat('s', 400) FROM generate_series(1, 3000) g; +CREATE TABLE ${p}joined_target (k int4 PRIMARY KEY, v text) $am; +INSERT INTO ${p}joined_target SELECT g, 't' FROM generate_series(1, 3000) g; +}); + my $plan = + "SET enable_hashjoin = off; SET enable_mergejoin = off; SET enable_material = off;"; + my $update = + "UPDATE ${p}joined_target t SET v = s.v || 'u' FROM ${p}joined_source s WHERE t.k = s.k"; + if ($am eq '') + { + my $updater = $node->background_psql('postgres'); + $updater->query_safe( + "$plan SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" + ); + $updater->query_until(qr/started/, "\\echo started\n$update;\n"); + $node->wait_for_event('client backend', 'doltgres-spill-slots-read'); + is( $node->safe_psql( + 'postgres', "SELECT doltgres_test_fold('joined_source')"), + 't', + "a fold under the held UPDATE ... FROM's source scan installs a base" + ); + launcher_freed(); + $node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" + ); + $updater->query_safe('SELECT 1'); + $updater->quit; + } + else + { + $node->safe_psql('postgres', "$plan $update"); + } + $joined{$p} = $node->safe_psql('postgres', + "SELECT count(*) || ' ' || md5(string_agg(k || v, ',' ORDER BY k)) FROM ${p}joined_target" + ); +} +is($joined{''}, $joined{heap_}, + "an UPDATE ... FROM whose source scan a fold's drop passes under updates every row as heap's twin's" +); + +# Bugs rows 240 and 241: a scan held after it read its storage's spill slots, +# while a fold installs a base that applies a committed transaction's spilled +# rows and drops them from the delta, moves to that base at its next read and +# reads every row once. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE read_source (v text) USING doltgres_twin; +INSERT INTO read_source SELECT 'r' || i || repeat('x', 400) FROM generate_series(1, 3000) i; +CREATE TABLE read_rows (v text); +}); +my $read_count = + q{SELECT count(*) FROM doltgres_test_scan('read_rows', false, -1)}; + +# held_scan starts a scan of read_rows in a session of its own: a cursor +# that has fetched its first 1,000 rows, held where its next fetch reads the +# storage's spill slots, and returns the session. +sub held_scan +{ + my $scan = $node->background_psql('postgres'); + $scan->query_safe('BEGIN; DECLARE c CURSOR FOR SELECT v FROM read_rows'); + $scan->query_safe('FETCH 1000 FROM c'); + $scan->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" + ); + $scan->query_until(qr/held/, "\\echo held\nFETCH ALL FROM c;\n"); + $node->wait_for_event('client backend', 'doltgres-spill-slots-read'); + return $scan; +} + +# released lets a held scan go on and returns the count of the rows it read, +# its first 1,000 with the rest. +sub released +{ + my ($scan) = @_; + $node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" + ); + my @rest = split /\n/, $scan->query_safe('SELECT 1'); + $scan->query_safe('COMMIT'); + $scan->quit; + return 1000 + @rest - 1; +} +launcher_held(); +$node->safe_psql('postgres', + q{SELECT count(*) FROM doltgres_test_insert('read_rows', 'read_source', true, 0)} +); +my $scan = held_scan(); +is($node->safe_psql('postgres', q{SELECT doltgres_test_fold('read_rows')}), + 't', 'a fold under the held scan installs a base'); +launcher_freed(); +is(released($scan), 3000, + 'a scan held across a fold that applied its spilled rows reads every row' +); +is($node->safe_psql('postgres', $read_count), + '3000', 'and a scan after it reads every row'); + +# Bugs rows 240 and 241's point reads: a fetch of a spilled row by its tuple id, +# held inside its read of the delta between its tuple id's lookup and its +# versions' read, and an index fetch of it by its row key, held inside its +# read of the delta, each while a fold installs a base holding the row and +# drops its spill, read the row, as the same fetch after the fold does. Each +# held case holds the fold launcher from before its batch's writes, so no fold +# worker the writes ask for applies the batch, or holds the horizon below it, +# before the case's own fold. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE point_source USING doltgres_twin AS SELECT 'p' || i || repeat('x', 400) AS v FROM generate_series(1, 3000) i; +CREATE TABLE point_rows (v text); +CREATE TABLE point_one AS SELECT 'p1' || repeat('x', 400) AS v; +}); +# copied writes the point table's batch of spilled rows and returns its first +# row's tuple id, which its first spill holds, and that id as a number. +sub copied +{ + my $tid = $node->safe_psql('postgres', + q{SELECT min(t) FROM doltgres_test_insert('point_rows', 'point_source', true, 0) t} + ); + return ( + $tid, + $node->safe_psql( + 'postgres', + "SELECT ((('$tid'::tid::text::point)[0])::int8 << 16) | (('$tid'::tid::text::point)[1])::int8" + )); +} +for my $name ('by tuple id', 'by row key') +{ + launcher_held(); + my ($tid, $copy) = copied(); + my $fetch = + $name eq 'by tuple id' + ? "SELECT doltgres_test_fetch('point_rows', '$tid')" + : "SELECT string_agg(r, ' ') FROM doltgres_test_index_fetch_key('point_rows', 'point_one', $copy) r"; + my $at = + $name eq 'by tuple id' + ? 'doltgres-fetch-keyed' + : 'doltgres-spill-slots-read'; + my $point = $node->background_psql('postgres'); + $point->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('$at', 'wait')" + ); + $point->query_until(qr/held/, "\\echo held\n$fetch;\n"); + $node->wait_for_event('client backend', $at); + is( $node->safe_psql( + 'postgres', q{SELECT doltgres_test_fold('point_rows')}), + 't', + "a fold under the held fetch $name installs a base"); + launcher_freed(); + $node->safe_psql('postgres', + "SELECT injection_points_detach('$at'); SELECT injection_points_wakeup('$at')" + ); + my $held = $point->query_safe('SELECT 1'); + $point->quit; + $held =~ s/\n1$//; + my $after = $node->safe_psql('postgres', $fetch); + like( + $after, + qr/^\(\d+,\d+\) \d+ p1x/, + "a fetch $name reads the row after the fold"); + is($held, $after, + "a fetch $name held across a fold that applied its spilled row reads it" + ); +} + +# Bugs rows 240 and 241's position kinds: each kind of read the test module +# reaches, held after it read its storage's spill slots while a fold installs a +# base that applies a committed transaction's spilled rows and drops them from +# the delta, moves to that base at its next read and reads every row once, as a +# read after the fold does and as many rows as heap's twin holds: a scan forward +# and backward over keyless copies, a tid range read forward and backward, a +# parallel scan's workers, and a scan of a table whose key a collation compares. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE kinds_source (v text) USING doltgres_twin; +INSERT INTO kinds_source SELECT repeat('r', 400) || (i % 500) FROM generate_series(1, 3000) i; +CREATE TABLE kinds (v text); +CREATE TABLE heap_kinds (v text) USING doltgres_twin; +CREATE TABLE compared (k text COLLATE "und-x-icu" PRIMARY KEY, v text); +CREATE TABLE heap_compared (k text COLLATE "und-x-icu" PRIMARY KEY, v text) USING doltgres_twin; +}); + +# held_kind starts a read in a session of its own, held at the point where it +# reads its storage's spill slots, and returns the session. +sub held_kind +{ + my ($query) = @_; + my $read = $node->background_psql('postgres'); + $read->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" + ); + $read->query_until(qr/held/, "\\echo held\n$query;\n"); + $node->wait_for_event('client backend', 'doltgres-spill-slots-read'); + return $read; +} + +# across_fold writes a batch of a table's spilled rows, holds a read of them, +# folds the table, lets the read go on, and returns its answer with the same +# read's answer after the fold, the twin's count, and whether the fold +# installed a base. +sub across_fold +{ + my ($table, $batch, $query, $twin) = @_; + launcher_held(); + $node->safe_psql('postgres', $batch); + my $read = held_kind($query); + my $folded = + $node->safe_psql('postgres', "SELECT doltgres_test_fold('$table')"); + launcher_freed(); + $node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" + ); + my $held = $read->query_safe('SELECT 1'); + $read->quit; + $held =~ s/\n1$//; + return ( + $held, + $node->safe_psql('postgres', $query), + $node->safe_psql('postgres', $twin), $folded); +} + +my $insert = + q{SELECT count(*) FROM doltgres_test_insert('kinds', 'kinds_source', true, 0); + INSERT INTO heap_kinds SELECT * FROM kinds_source}; +my $twin = q{SELECT count(*) FROM heap_kinds}; +my %kind = ( + forward => + q{SELECT count(*) || ' ' || md5(string_agg(substring(r from '^\S+ \S+ (.*)$'), ',' ORDER BY r)) FROM doltgres_test_scan('kinds', false, -1) r}, + backward => + q{SELECT count(*) || ' ' || md5(string_agg(substring(r from '^\S+ \S+ (.*)$'), ',' ORDER BY r)) FROM doltgres_test_scan('kinds', true, -1) r}, + 'tid range forward' => + q{SELECT count(*) || ' ' || md5(string_agg(substring(r from '^\S+ \S+ (.*)$'), ',' ORDER BY r)) FROM doltgres_test_tidrange('kinds', '(0,1)', '(4294967294,0)', false) r}, + 'tid range backward' => + q{SELECT count(*) || ' ' || md5(string_agg(substring(r from '^\S+ \S+ (.*)$'), ',' ORDER BY r)) FROM doltgres_test_tidrange('kinds', '(0,1)', '(4294967294,0)', true) r}, + parallel => + q{SELECT count(*) || ' ' || md5(string_agg(substring(r from '^\S+ \S+ (.*)$'), ',' ORDER BY r)) FROM doltgres_test_parallel('kinds', 3, false, 0, NULL, NULL) r}, +); +for my $name ( + 'forward', 'backward', + 'tid range forward', 'tid range backward', + 'parallel') +{ + my ($held, $after, $count, $folded) = + across_fold('kinds', $insert, $kind{$name}, $twin); + is($folded, 't', "a fold under the held $name read installs a base"); + is($held, $after, "a $name read held across a fold reads every row once"); + like($held, qr/^$count /, "and as many rows as heap's twin holds"); +} +my @compared = across_fold( + 'compared', + q{SELECT doltgres_test_write('compared', true, 'INSERT INTO compared SELECT ''k'' || i, repeat(''v'', 400) FROM generate_series(1, 3000) i'); + INSERT INTO heap_compared SELECT 'k' || i, repeat('v', 400) FROM generate_series(1, 3000) i}, + q{SELECT count(*) || ' ' || md5(string_agg(substring(r from '^\S+ \S+ (.*)$'), ',' ORDER BY r)) FROM doltgres_test_scan('compared', false, -1) r}, + q{SELECT count(*) FROM heap_compared}); +is($compared[3], 't', + "a fold under the held read of a table whose key a collation compares installs a base" +); +is($compared[0], $compared[1], + "a read of a table whose key a collation compares, held across a fold, reads every row once" +); +like( + $compared[0], + qr/^$compared[2] /, + "and as many rows as heap's twin holds"); + +# Bugs rows 240 and 241's position kinds through a fork btree index: an index +# scan forward and backward, an index-only scan, and a bitmap scan, each held +# after it read a storage's spill slots while a fold of the index installs a +# base that applies a committed transaction's spilled entries, read every row +# once, as heap's twin gives them, with many rows of one key spanning the fold. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE ix (id int4, t text); +CREATE INDEX ix_t ON ix (t); +CREATE TABLE heap_ix (id int4, t text) USING doltgres_twin; +CREATE INDEX heap_ix_t ON heap_ix (t); +CREATE TABLE ix_source USING doltgres_twin AS SELECT i AS id, repeat('v', 200) || (i % 41) AS t FROM generate_series(1, 2000) i; +INSERT INTO heap_ix SELECT * FROM ix_source; +SELECT doltgres_test_insert('ix', 'ix_source', false, 0); +SELECT doltgres_test_fold('ix'); +SELECT doltgres_test_index_build('ix', 'ix_t'); +ANALYZE heap_ix; +}); +my $routine = + "DO \$\$ BEGIN PERFORM doltgres_test_index_routine('ix_t'); END \$\$;"; +my $indexes = + 'SET enable_seqscan = off; SET enable_bitmapscan = off; SET enable_indexonlyscan = off;'; +my %plans = ( + 'index forward' => + [ $indexes, q{SELECT id FROM %T WHERE t >= 'v' ORDER BY t} ], + 'index backward' => + [ $indexes, q{SELECT id FROM %T WHERE t >= 'v' ORDER BY t DESC} ], + 'index-only' => [ + 'SET enable_seqscan = off; SET enable_bitmapscan = off;', + q{SELECT t FROM %T WHERE t >= 'v' ORDER BY t} + ], + bitmap => [ + 'SET enable_seqscan = off; SET enable_indexscan = off; SET enable_indexonlyscan = off;', + q{SELECT id FROM %T WHERE t >= 'v'} + ],); +my $next = 2001; +for my $name ('index forward', 'index backward', 'index-only', 'bitmap') +{ + my ($settings, $query) = @{ $plans{$name} }; + my ($from, $to) = ($next, $next + 3999); + $next = $to + 1; + launcher_held(); + $node->safe_psql( + 'postgres', + "$routine SELECT doltgres_test_write('ix', true, 'INSERT INTO ix SELECT i, repeat(''v'', 200) || (i % 41) FROM generate_series($from, $to) i'); + INSERT INTO heap_ix SELECT i, repeat('v', 200) || (i % 41) FROM generate_series($from, $to) i" + ); + (my $fork = $query) =~ s/%T/ix/g; + $fork =~ s/'/''/g; + (my $heap = $query) =~ s/%T/heap_ix/g; + my $read = $node->background_psql('postgres'); + $read->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" + ); + $read->query_until(qr/held/, + "\\echo held\n$settings $routine SELECT doltgres_test_standing('ix', '$fork');\n" + ); + $node->wait_for_event('client backend', 'doltgres-spill-slots-read'); + my $entries = + $node->safe_psql('postgres', "SELECT doltgres_test_index_base('ix_t')"); + isnt( + $node->safe_psql( + 'postgres', "SELECT doltgres_test_index_fold('ix_t')"), + $entries, + "a fold of the index under the held $name read installs a base"); + is($node->safe_psql('postgres', "SELECT doltgres_test_fold('ix')"), + 't', "a fold of the table under the held $name read installs a base"); + launcher_freed(); + $node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" + ); + my $held = $read->query_safe('SELECT 1'); + $read->quit; + $held =~ s/\n1$//; + my $twin = $node->safe_psql('postgres', "$settings $heap"); + is( join(',', sort split /\n/, $held), + join(',', sort split /\n/, $twin), + "a fork btree's $name read held across a fold of its index reads every row once" + ); +} + +# Bugs rows 240 and 315: a change by tuple id, held where it looks up its +# row's key, before a fold installs a base holding the row and drops the +# row's versions from the delta, finds the row's key and changes it, as the +# same change does with no fold: an UPDATE after an INSERT in its own +# transaction, a DELETE, and a row lock. +my %changes = ( + update => [ + q{BEGIN; INSERT INTO %T SELECT g, 'y' FROM generate_series(2001, 5000) g}, + q{WITH c AS (UPDATE %T SET v = 'z' WHERE id <= 1000 RETURNING 1) SELECT count(*) FROM c}, + 'COMMIT' + ], + delete => [ + 'BEGIN', + 'WITH c AS (DELETE FROM %T WHERE id <= 1000 RETURNING 1) SELECT count(*) FROM c', + 'COMMIT' + ], + lock => [ + 'BEGIN', + 'SELECT count(*) FROM (SELECT id FROM %T WHERE id <= 1000 FOR UPDATE) l', + 'COMMIT' + ],); +my %change_named = ( + update => 'an UPDATE', + delete => 'a DELETE', + lock => 'a row lock'); +for my $name ('update', 'delete', 'lock') +{ + my ($before, $change, $after) = @{ $changes{$name} }; + my $named = $change_named{$name}; + my $table = "changed_$name"; + s/%T/$table/g for ($before, $change, $after); + launcher_held(); + $node->safe_psql('postgres', + "CREATE TABLE $table (id int4 PRIMARY KEY, v text); INSERT INTO $table SELECT g, repeat('x', 400) FROM generate_series(1, 2000) g" + ); + my $changer = $node->background_psql('postgres', on_error_stop => 0); + $changer->query_safe($before); + $changer->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-tuple-keyed', 'wait')" + ); + $changer->query_until(qr/held/, "\\echo held\n$change;\n"); + $node->wait_for_event('client backend', 'doltgres-tuple-keyed'); + is($node->safe_psql('postgres', "SELECT doltgres_test_fold('$table')"), + 't', "a fold under $named by tuple id, held, installs a base"); + launcher_freed(); + $node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-tuple-keyed'); SELECT injection_points_wakeup('doltgres-tuple-keyed')" + ); + my $changed = $changer->query('SELECT 1'); + $changed =~ s/\n1$//; + $changed .= " $changer->{stderr}" if $changer->{stderr} =~ /ERROR/; + $changer->query($after); + $changer->quit; + is($changed, '1000', + "$named by tuple id held across a fold that applied its rows changes every row" + ); +} +$node->stop; + +done_testing(); diff --git a/src/doltgres/test/t/072_standby_spills.pl b/src/doltgres/test/t/072_standby_spills.pl new file mode 100644 index 0000000000000..31f310dcbf3b8 --- /dev/null +++ b/src/doltgres/test/t/072_standby_spills.pl @@ -0,0 +1,205 @@ +# Copyright 2026 Dolthub, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A streaming standby's replay of spilled batches (bugs row 298): a primary +# writes a committed batch of rows past its delta_memory, which it spills, and +# folds it while a scan of it is held at its read of the spill slots; writes +# two batches into another table, each folded while a point fetch of a spilled +# row is held the same way; and writes a second batch into the first table, +# which the standby replays, folds at its primary's point while its own scan is +# held, and reads whole. Before, the standby's redo of the second batch's row +# changes died with "a location of the delta names no record its hold +# reaches". + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $reading = PostgreSQL::Test::Cluster->new('spill_reading'); +$reading->init(allows_streaming => 1); +$reading->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\nmax_parallel_workers_per_gather = 0" +); +$reading->start; +$reading->safe_psql( + 'postgres', q{ +CREATE EXTENSION injection_points; +CREATE EXTENSION doltgres_test; +CREATE TABLE read_source (v text) USING doltgres_twin; +INSERT INTO read_source SELECT 'r' || i FROM generate_series(1, 100000) i; +CREATE TABLE read_rows (v text); +}); +$reading->backup('reading_backup'); +my $reading_standby = PostgreSQL::Test::Cluster->new('spill_reading_standby'); +$reading_standby->init_from_backup($reading, 'reading_backup', + has_streaming => 1); +$reading_standby->append_conf('postgresql.conf', + "max_parallel_workers_per_gather = 0"); +$reading_standby->start; +my $read_count = + q{SELECT count(*) FROM doltgres_test_scan('read_rows', false, -1)}; + +# held_scan starts a scan of read_rows on a node in a session of its own, held at +# the point where it reads the storage's spill slots, and returns the session. +sub held_scan +{ + my ($on) = @_; + my $scan = $on->background_psql('postgres'); + $scan->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" + ); + $scan->query_until(qr/held/, "\\echo held\n$read_count;\n"); + $on->wait_for_event('client backend', 'doltgres-spill-slots-read'); + return $scan; +} + +# released lets a node's held scan go on and returns its count. +sub released +{ + my ($on, $scan) = @_; + $on->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" + ); + my $count = $scan->query_safe('SELECT 1'); + $scan->quit; + return $count; +} +$reading->safe_psql( + 'postgres', + q{SELECT count(*) FROM doltgres_test_insert('read_rows', 'read_source', true, 0)}, + timeout => 1200); +$reading->wait_for_catchup($reading_standby); +my $scan = held_scan($reading); +$reading->safe_psql('postgres', q{SELECT doltgres_test_fold('read_rows')}); +released($reading, $scan); +is($reading->safe_psql('postgres', $read_count), + '100000', 'and a scan after it reads every row'); + +# A fetch of a spilled row by its tuple id and an index fetch of it by its row +# key, each held inside its read of the delta while a fold installs a base +# holding the row and drops its spill, then read again. +$reading->safe_psql( + 'postgres', q{ +CREATE TABLE point_source USING doltgres_twin AS SELECT 'p' || i AS v FROM generate_series(1, 100000) i; +CREATE TABLE point_rows (v text); +CREATE TABLE point_one AS SELECT 'p1' AS v; +}); +# copied writes the point table's batch of spilled rows and returns its first +# row's tuple id, which its first spill holds, and that id as a number. +sub copied +{ + my $tid = $reading->safe_psql( + 'postgres', + q{SELECT min(t) FROM doltgres_test_insert('point_rows', 'point_source', true, 0) t}, + timeout => 1200); + return ( + $tid, + $reading->safe_psql( + 'postgres', + "SELECT ((('$tid'::tid::text::point)[0])::int8 << 16) | (('$tid'::tid::text::point)[1])::int8" + )); +} +for my $name ('by tuple id', 'by row key') +{ + my ($tid, $copy) = copied(); + my $fetch = + $name eq 'by tuple id' + ? "SELECT doltgres_test_fetch('point_rows', '$tid')" + : "SELECT string_agg(r, ' ') FROM doltgres_test_index_fetch_key('point_rows', 'point_one', $copy) r"; + my $point = $reading->background_psql('postgres'); + $point->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-spill-slots-read', 'wait')" + ); + $point->query_until(qr/held/, "\\echo held\n$fetch;\n"); + $reading->wait_for_event('client backend', 'doltgres-spill-slots-read'); + $reading->safe_psql('postgres', + q{SELECT doltgres_test_fold('point_rows')}); + $reading->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-spill-slots-read'); SELECT injection_points_wakeup('doltgres-spill-slots-read')" + ); + $point->query_safe('SELECT 1'); + $point->quit; + isnt($reading->safe_psql('postgres', $fetch), + '', "a fetch $name reads the row after the fold"); +} +$reading->safe_psql( + 'postgres', + q{SELECT count(*) FROM doltgres_test_insert('read_rows', 'read_source', true, 0)}, + timeout => 1200); +$reading->wait_for_catchup($reading_standby); +$scan = held_scan($reading_standby); +$reading->safe_psql('postgres', q{SELECT doltgres_test_fold('read_rows')}); +# A standby folds a byte-ordered table's reached point only as the next record +# arrives (bugs row 334, whose fix takes this record out), so one follows the +# fold, and the standby's own fold runs until its base is its primary's, since +# one run after its catch-up isn't always enough (bugs row 337, whose fix takes +# the retry out). +$reading->safe_psql('postgres', 'SELECT txid_current()'); +$reading->wait_for_catchup($reading_standby); +my $points = q{SELECT doltgres_test_base_sizes('read_rows')}; +my $primary_points = $reading->safe_psql('postgres', $points); +ok( $reading_standby->poll_query_until( + 'postgres', + "SELECT doltgres_test_base_sizes('read_rows') = '$primary_points' FROM (SELECT doltgres_test_fold('read_rows')) f" + ), + "the standby folded at its primary's point while its scan was held"); +released($reading_standby, $scan); +is($reading_standby->safe_psql('postgres', $read_count), + '200000', "and the standby's scan after it reads every row"); +$reading_standby->stop; +$reading->stop; + +# Bugs row 339: one INSERT past a 1MB delta_memory finishes, a wide one and a +# narrow one, and the 1MB standby replays each; before, the primary's writer +# waited at the budget for good. +my $small = PostgreSQL::Test::Cluster->new('spill_small'); +$small->init(allows_streaming => 1); +$small->append_conf('postgresql.conf', + "delta_memory = 1MB\nfold_threshold = 1GB\nfold_timeout = 3600s"); +$small->start; +$small->backup('small_backup'); +my $small_standby = PostgreSQL::Test::Cluster->new('spill_small_standby'); +$small_standby->init_from_backup($small, 'small_backup', has_streaming => 1); +$small_standby->start; + +foreach my $case ([ 'wide_rows', 3000, q{repeat('w', 600)} ], + [ 'narrow_rows', 20000, q{'n'} ]) +{ + my ($table, $rows, $value) = @$case; + my (undef, $out) = $small->psql( + 'postgres', qq{ +SET statement_timeout = '180s'; +CREATE TABLE $table (k int4 PRIMARY KEY, v text) WITH (fillfactor = 50); +INSERT INTO $table SELECT g, $value FROM generate_series(1, $rows) g; +SELECT count(*) FROM $table; +}); + is($out, $rows, + "an INSERT of $rows rows into $table past a 1MB delta_memory finishes" + ); + $small->wait_for_catchup($small_standby); + is( $small_standby->safe_psql('postgres', "SELECT count(*) FROM $table"), + $rows, + "and the 1MB standby replays it"); +} +$small_standby->stop; +$small->stop; + +done_testing(); diff --git a/src/doltgres/test/t/106_cport_simc.pl b/src/doltgres/test/t/106_cport_simc.pl index b6d8a20e7049e..91eaac8f6f312 100644 --- a/src/doltgres/test/t/106_cport_simc.pl +++ b/src/doltgres/test/t/106_cport_simc.pl @@ -37,6 +37,7 @@ 'simc::tests::a_checkpoint_sets_the_redo_point_the_host_answers', 'simc::tests::a_crash_fails_each_call_until_a_recovery_takes_the_files_back_to_their_syncs', 'simc::tests::a_delta_page_is_taken_written_and_given_back_as_the_simulations_host', + 'simc::tests::a_delta_page_takes_one_of_the_directorys_free_slots', 'simc::tests::a_dictionary_simc_prepares_compresses_and_decompresses_and_drops_in_the_c', 'simc::tests::a_failed_allocation_of_a_listings_names_frees_the_names_made_and_raises_out_of_memory', 'simc::tests::a_failed_allocation_of_a_panics_payload_frees_its_message_and_raises_out_of_memory', @@ -44,8 +45,10 @@ 'simc::tests::a_free_pass_is_counted', 'simc::tests::a_hinted_c_error_crosses_to_sim_as_its_literals', 'simc::tests::a_historic_snapshot_lends_its_bounds_command_and_sorted_xids', + 'simc::tests::a_hold_of_the_test_process_keeps_its_data_directory_past_its_capability', 'simc::tests::a_linked_delta_page_opens_reads_appends_seals_and_retires_as_the_simulations_host', 'simc::tests::a_nap_is_counted_and_the_disk_space_set_is_answered', + 'simc::tests::a_non_vacuumable_snapshot_finds_xids_below_its_horizon_removable', 'simc::tests::a_pause_a_warning_a_log_and_the_last_redo_answer_as_the_simulations_host', 'simc::tests::a_pooled_records_doors_answer_as_the_simulations_host', 'simc::tests::a_processs_flags_are_cleared_before_each_exit_work', @@ -54,6 +57,7 @@ 'simc::tests::a_refused_open_and_a_refused_record_fail_once_with_the_refusal', 'simc::tests::a_refused_take_fails_once_and_refuse_nothing_takes_back_the_rest', 'simc::tests::a_reports_debug_is_the_rusts', + 'simc::tests::a_reports_trace_is_the_rusts', 'simc::tests::a_returning_actor_ends_ok_and_its_endings_compare_as_the_rusts', 'simc::tests::a_run_check_fails_the_run_at_a_yield_with_its_message', 'simc::tests::a_run_checks_c_panic_fails_the_run_with_its_message', @@ -89,7 +93,10 @@ 'simc::tests::the_delta_advance_seed_and_pass_doors_answer_as_the_simulations_host', 'simc::tests::the_delta_hold_and_budget_doors_answer_as_the_simulations_host', 'simc::tests::the_epoch_doors_answer_as_the_simulations_host', + 'simc::tests::the_injected_door_answers_as_the_simulations_host', 'simc::tests::the_key_doors_answer_as_the_simulations_host', + 'simc::tests::the_logged_records_are_the_records_the_wal_took', + 'simc::tests::the_new_cid_door_logs_its_record_as_the_simulations_host', 'simc::tests::the_next_decompression_runs_out_of_memory_when_asked', 'simc::tests::the_node_cache_installs_finds_and_unpins_a_nodes_frame', 'simc::tests::the_test_only_std_fs_lists_a_directorys_names', @@ -98,6 +105,7 @@ 'simc::tests::the_test_only_std_fs_removes_a_directory_with_its_contents', 'simc::tests::the_test_only_std_fs_writes_reads_appends_and_removes_a_file', 'simc::tests::the_wal_a_checkpoint_and_a_databases_quiet_answer_as_the_simulations_host', + 'simc::tests::the_wal_flush_door_answers_as_the_simulations_host', ); my @tests = split /\n/, $rows; is_deeply([ map { (split /\t/, $_, 2)[0] } @tests ], diff --git a/src/doltgres/test/t/124_carry_unlisted.pl b/src/doltgres/test/t/124_carry_unlisted.pl new file mode 100644 index 0000000000000..c8c6c42d6c0a4 --- /dev/null +++ b/src/doltgres/test/t/124_carry_unlisted.pl @@ -0,0 +1,332 @@ +# Copyright 2026 Dolthub, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A carry whose count's add fails takes its files back out (bugs row 291's +# review R2): a VACUUM FULL carries three writers' recently dead versions, +# which an older snapshot keeps, into spill files under each writer, and the +# add of the second file's count fails; the carry takes its files back out and +# gives back the first add, so every writer's count is where it was, and a +# restart replays the carry's records and its unlist to nothing. A failure +# inside the unlist's window stops the server, and the restart's recount +# leaves every count at the versions its storages hold. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('carrier'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\nrestart_after_crash = on\ncheckpoint_timeout = 1h" +); +$node->start; +$node->safe_psql( + 'postgres', q{ +CREATE EXTENSION injection_points; +CREATE EXTENSION doltgres_test; +CREATE TABLE carried (k int4 PRIMARY KEY, v text) WITH (fillfactor = 50); +INSERT INTO carried SELECT g, repeat('c', 20) FROM generate_series(1, 30) g; +SELECT doltgres_test_fold('carried'); +}); + +# Every server takes no checkpoint or restartpoint by time: a standby's +# restartpoint moves the writers' counts while the rounds compare them. +# counts returns each writer's entry's count on a server, the primary when +# none is given, -1 for one with none. +sub counts +{ + my ($writers, $on) = @_; + $on //= $node; + return join ' ', map { + $on->safe_psql('postgres', + "SELECT doltgres_test_status_table(1, $_, 0)") + } @$writers; +} + +# updated runs three updates, each its own transaction, while an older +# snapshot holds their older versions, and returns the writers' xids. +sub updated +{ + my ($holder) = @_; + my @writers; + for my $n (1 .. 3) + { + push @writers, + $node->safe_psql('postgres', + "BEGIN; UPDATE carried SET v = v || 'u' WHERE mod(k, 3) = " + . ($n - 1) + . "; SELECT txid_current(); COMMIT;"); + } + return \@writers; +} + +my $holder = $node->background_psql('postgres'); +$holder->query_safe( + 'BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM pg_class'); +my $writers = updated($holder); +my $before = counts($writers); + +$node->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-carry-counting', 'wait')"); +my $rewriter = $node->background_psql('postgres', on_error_stop => 0); +$rewriter->query_until(qr/rewriting/, + "\\echo rewriting\nVACUUM FULL carried;\n"); +$node->wait_for_event('client backend', 'doltgres-carry-counting'); +$node->safe_psql( + 'postgres', q{ +SELECT injection_points_detach('doltgres-carry-counting'); +SELECT injection_points_attach('doltgres-carry-counting', 'error'); +SELECT injection_points_wakeup('doltgres-carry-counting'); +}); +$rewriter->quit; +$node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-carry-counting')"); +is(counts($writers), $before, + "a carry whose second count's add fails gives back the first, every writer's count where it was" +); +$holder->query_safe('COMMIT'); +$holder->quit; + +$node->restart; +is(counts($writers), $before, + "a restart replays the carry's records and its unlist to nothing"); +is($node->safe_psql('postgres', 'SELECT count(*) FROM carried'), + '30', + "the table's rows read back after the failed carry and the restart"); + +# A concurrent build's late entries (bugs row 291's review R2): CREATE INDEX +# CONCURRENTLY carries the entries of rows whose writers are past its horizon, +# which an older snapshot holds, into spill files under each writer, in an +# index whose storage an earlier transaction made, so the build's abort +# drops nothing. With the second count's add failed, the build takes its +# files back out and logs the unlist, and a standby that spilled the carried +# entries, too large for its delta_memory, before the unlist arrived takes +# them out of its spill files too: its delta of the index holds its primary's +# entries, and every writer's count is where it was. +$node->backup('late'); +my $standby = PostgreSQL::Test::Cluster->new('late_standby'); +$standby->init_from_backup($node, 'late', has_streaming => 1); +$standby->append_conf('postgresql.conf', + "delta_memory = 1MB\nfold_threshold = 1GB\nfold_timeout = 3600s\ncheckpoint_timeout = 1h" +); +$standby->start; +$node->safe_psql('postgres', 'CREATE TABLE late (k int4, v text)'); +$holder = $node->background_psql('postgres'); +$holder->query_safe( + 'BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM pg_class'); +my @late_writers; + +for my $n (0 .. 2) +{ + my $from = 3000 * $n + 1; + push @late_writers, + $node->safe_psql('postgres', + "BEGIN; INSERT INTO late SELECT g, md5(g::text) || repeat('l', 200) FROM generate_series($from, $from + 2999) g; SELECT txid_current(); COMMIT;" + ); +} +$before = counts(\@late_writers); +$node->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-carry-counting', 'wait')"); +my $builder = $node->background_psql('postgres', on_error_stop => 0); +$builder->query_until(qr/building/, + "\\echo building\nCREATE INDEX CONCURRENTLY late_v ON late (v COLLATE \"C\");\n" +); +$node->wait_for_event('client backend', 'doltgres-carry-counting'); +$node->safe_psql( + 'postgres', q{ +SELECT injection_points_detach('doltgres-carry-counting'); +SELECT injection_points_attach('doltgres-carry-counting', 'error'); +SELECT injection_points_wakeup('doltgres-carry-counting'); +}); +$builder->quit; +$node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-carry-counting')"); +$holder->query_safe('COMMIT'); +$holder->quit; +is(counts(\@late_writers), $before, + "a concurrent build whose second count's add fails gives back the first, every writer's count where it was" +); +$node->wait_for_replay_catchup($standby); +is(counts(\@late_writers, $standby), + $before, + "the standby replays the unlist, every writer's count where it was"); +is( $standby->safe_psql( + 'postgres', "SELECT md5(doltgres_test_index_entries('late_v'))"), + $node->safe_psql( + 'postgres', "SELECT md5(doltgres_test_index_entries('late_v'))"), + "the standby's delta of the index holds its primary's entries, the carried ones out" +); +is($standby->safe_psql('postgres', 'SELECT count(*) FROM late'), + '9000', "the standby reads the table whole"); +$standby->stop; + +# A plain rewrite's failed carry on a standby (bugs row 291's review G1): a +# VACUUM FULL carries three writers' recently dead rows, with their tuple ids +# and their puts' takes, which an older snapshot holds, and the add of the +# second file's count fails; a standby that replayed the carried rows takes +# them back out at the unlist, so every writer's count there is where it was. +# The new storage itself goes with the rewrite's abort, on the standby too. +# The standby has the default delta_memory: at 1MB its replay of so wide a +# carry stalls at the budget (bugs rows 332 and 304). +$node->backup('wide'); +my $wide_standby = PostgreSQL::Test::Cluster->new('wide_standby'); +$wide_standby->init_from_backup($node, 'wide', has_streaming => 1); +$wide_standby->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\nallow_in_place_tablespaces = on\ncheckpoint_timeout = 1h" +); +$wide_standby->start; +$node->safe_psql( + 'postgres', q{ +CREATE TABLE carried_wide (k int4 PRIMARY KEY, v text) WITH (fillfactor = 50); +INSERT INTO carried_wide SELECT g, repeat('w', 600) FROM generate_series(1, 3000) g; +SELECT doltgres_test_fold('carried_wide'); +}); +$node->wait_for_replay_catchup($wide_standby); +$holder = $node->background_psql('postgres'); +$holder->query_safe( + 'BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM pg_class'); +my @wide_writers; + +for my $n (0 .. 2) +{ + push @wide_writers, + $node->safe_psql('postgres', + "BEGIN; UPDATE carried_wide SET v = v || 'u' WHERE mod(k, 3) = $n; SELECT txid_current(); COMMIT;" + ); +} +$before = counts(\@wide_writers); +$node->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-carry-counting', 'wait')"); +$rewriter = $node->background_psql('postgres', on_error_stop => 0); +$rewriter->query_until(qr/widening/, + "\\echo widening\nVACUUM FULL carried_wide;\n"); +$node->wait_for_event('client backend', 'doltgres-carry-counting'); +$node->safe_psql( + 'postgres', q{ +SELECT injection_points_detach('doltgres-carry-counting'); +SELECT injection_points_attach('doltgres-carry-counting', 'error'); +SELECT injection_points_wakeup('doltgres-carry-counting'); +}); +$rewriter->quit; +$node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-carry-counting')"); +$holder->query_safe('COMMIT'); +$holder->quit; +is(counts(\@wide_writers), $before, + "a wide table's carry whose second count's add fails gives back the first, every writer's count where it was" +); +$node->wait_for_replay_catchup($wide_standby); +is( counts(\@wide_writers, $wide_standby), + $before, + "a standby that replayed the carried rows takes them out at the unlist, every writer's count where it was" +); +is( $wide_standby->safe_psql( + 'postgres', 'SELECT count(*), sum(length(v)) FROM carried_wide'), + $node->safe_psql( + 'postgres', 'SELECT count(*), sum(length(v)) FROM carried_wide'), + "the standby reads the wide table as its primary does"); + +# A move's failed carry on a standby (bugs row 291's review G2): ALTER TABLE +# SET TABLESPACE carries its old storage's versions, with the free space map +# versions three writers' updates made, which an older snapshot holds, and +# the add of the second file's count fails; the standby takes out each +# carried map version by its record's end, as well as the carried rows' +# takes, so every writer's count there is where it was. +$node->safe_psql( + 'postgres', q{ +SET allow_in_place_tablespaces = on; +CREATE TABLESPACE moved_space LOCATION ''; +CREATE TABLE carried_moved (k int4 PRIMARY KEY, v text) WITH (fillfactor = 50); +INSERT INTO carried_moved SELECT g, repeat('m', 600) FROM generate_series(1, 3000) g; +SELECT doltgres_test_fold('carried_moved'); +}); +$holder = $node->background_psql('postgres'); +$holder->query_safe( + 'BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM pg_class'); +my @moved_writers; +for my $n (0 .. 2) +{ + push @moved_writers, + $node->safe_psql('postgres', + "BEGIN; UPDATE carried_moved SET v = v || 'u' WHERE mod(k, 3) = $n; SELECT txid_current(); COMMIT;" + ); +} +$before = counts(\@moved_writers); +$node->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-carry-counting', 'wait')"); +$rewriter = $node->background_psql('postgres', on_error_stop => 0); +$rewriter->query_until(qr/moving/, + "\\echo moving\nALTER TABLE carried_moved SET TABLESPACE moved_space;\n"); +$node->wait_for_event('client backend', 'doltgres-carry-counting'); +$node->safe_psql( + 'postgres', q{ +SELECT injection_points_detach('doltgres-carry-counting'); +SELECT injection_points_attach('doltgres-carry-counting', 'error'); +SELECT injection_points_wakeup('doltgres-carry-counting'); +}); +$rewriter->quit; +$node->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-carry-counting')"); +$holder->query_safe('COMMIT'); +$holder->quit; +is(counts(\@moved_writers), $before, + "a move's carry whose second count's add fails gives back the first, every writer's count where it was" +); +$node->wait_for_replay_catchup($wide_standby); +is( counts(\@moved_writers, $wide_standby), + $before, + "a standby that replayed the move's carried map versions takes them out at the unlist, every writer's count where it was" +); +$wide_standby->stop; + +# A failure inside the unlist's window, Postgres' critical section, stops the +# server after the unlist's record is flushed, and its restart recounts what +# its storages hold. +$holder = $node->background_psql('postgres', on_error_stop => 0); +$holder->query_safe( + 'BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM pg_class'); +$writers = updated($holder); +$before = counts($writers); +$node->safe_psql( + 'postgres', q{ +SELECT injection_points_attach('doltgres-carry-counting', 'error'); +SELECT injection_points_attach('doltgres-carry-unlisting', 'error'); +}); +my ($ret, $stdout, $stderr) = $node->psql('postgres', 'VACUUM FULL carried'); +like( + $stderr, + qr/server closed the connection unexpectedly|terminating connection/, + "a failure inside the unlist's window stops the server, as a PANIC does"); +ok($node->poll_query_until('postgres', 'SELECT 1', '1'), + "the server restarts after the PANIC"); +# The stopped rewrite's new storage is an orphan its crashed transaction left, +# as Postgres leaves an orphaned relation file, whose drop with its counts is +# bugs row 322's; the recount is what the storages list, so a second restart +# recounts the same. +my $recounted = counts($writers); +$node->restart; +is(counts($writers), $recounted, + "the restart after a stop inside the window recounts the versions its storages list, as a second restart does" +); +is($node->safe_psql('postgres', 'SELECT count(*) FROM carried'), + '30', "the table's rows read back after the stop and the restart"); + +done_testing(); diff --git a/src/doltgres/test/t/131_scan_costs.pl b/src/doltgres/test/t/131_scan_costs.pl new file mode 100644 index 0000000000000..fbd6b39129951 --- /dev/null +++ b/src/doltgres/test/t/131_scan_costs.pl @@ -0,0 +1,331 @@ +# Copyright 2026 Dolthub, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The table's part of a fork table's index and bitmap scan costs (design item +# 16; bugs row 312), each term read back through EXPLAIN's costs: an +# index-only scan reads no row of the table; the primary key's own index reads +# its leaves, which the node cache holds here; a secondary index's entry costs +# a descent of the table's tree, a comparison a level above the leaves, and, +# past the node cache, Mackert and Lohman's leaves at random; a keyless +# table's entry descends by the row's hash the same way; and a bitmap scan's +# row costs a tuple id resolution and a descent. + +use strict; +use warnings FATAL => 'all'; +use POSIX qw(ceil); +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('costs'); +$node->init; +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION doltgres_test'); +$node->safe_psql( + 'postgres', q{ +CREATE TABLE costed (id int4 PRIMARY KEY, v int4, pad text); +INSERT INTO costed SELECT g, g, repeat('x', 40) FROM generate_series(1, 20000) g; +CREATE INDEX costed_v ON costed (v); +CREATE TABLE keyless (v int4, pad text); +INSERT INTO keyless SELECT g, repeat('x', 40) FROM generate_series(1, 20000) g; +CREATE INDEX keyless_v ON keyless (v); +ANALYZE costed, keyless}); + +# planned returns a statement's plan's total cost and its scan's rows, under +# settings that leave the scan of the kind asked for. +sub planned +{ + my ($sql, $settings) = @_; + my $json = $node->safe_psql('postgres', + "SET enable_seqscan = off; $settings; EXPLAIN (FORMAT JSON) $sql"); + my ($total) = $json =~ /"Total Cost": ([0-9.]+)/; + my ($rows) = $json =~ /"Plan Rows": ([0-9]+)/; + return ($total, $rows); +} + +# shape returns a fork table's tree's leaves and height, and its tuple id +# tree's, as its scans' costs read them. +sub shape +{ + my ($table) = @_; + return split / /, + $node->safe_psql('postgres', + "SELECT doltgres_test_scan_shape('$table')"); +} + +# close_to says whether two costs agree within the 0.01 EXPLAIN rounds to. +sub close_to +{ + my ($got, $wanted) = @_; + return abs($got - $wanted) < 0.015; +} + +my $cpu_operator = 0.0025; +my $cpu_tuple = 0.01; +my $index_only = 'SET enable_bitmapscan = off'; +my $index_scan = + 'SET enable_bitmapscan = off; SET enable_indexonlyscan = off'; + +# A secondary index whose table's leaves the node cache holds: the index scan +# costs the index-only scan's plus a descent's comparisons a row. +my ($leaves, $height, $tid_leaves, $tid_height) = shape('costed'); +my ($only, $rows) = + planned('SELECT v FROM costed WHERE v < 1000', $index_only); +my ($scan) = planned('SELECT v FROM costed WHERE v < 1000', $index_scan); +ok( close_to($scan - $only, $rows * $height * $cpu_operator), + "a secondary index scan in the node cache costs a descent a row ($scan - $only, $rows rows, height $height)" +); + +# The primary key's own index, whose leaves are the table's rows and fit the +# node cache, reads no more of the table than its index-only scan. +my ($key_only) = + planned('SELECT id FROM costed WHERE id < 1000', $index_only); +my ($key_scan) = + planned('SELECT id FROM costed WHERE id < 1000', $index_scan); +ok( close_to($key_scan, $key_only), + "the primary key's own index scan in the node cache costs its index-only scan's ($key_scan, $key_only)" +); + +# A bitmap heap scan's rows each cost a tuple id resolution and a descent, +# beside the recheck's CPU and cost_bitmap_tree_node's tenth of an operator +# for handling the bitmap, over its bitmap index scan's cost. +my $bitmap = $node->safe_psql('postgres', + 'SET enable_seqscan = off; SET enable_indexscan = off; EXPLAIN (FORMAT JSON) SELECT pad FROM costed WHERE v < 1000' +); +my ($heap_total, $index_total) = + $bitmap =~ /"Total Cost": ([0-9.]+).*"Total Cost": ([0-9.]+)/s; +my ($bitmap_rows) = $bitmap =~ /"Plan Rows": ([0-9]+)/; +ok( close_to( + $heap_total - $index_total, + $bitmap_rows * ( + ($tid_height + $height) * $cpu_operator + + $cpu_tuple + + $cpu_operator + + 0.1 * $cpu_operator)), + "a bitmap heap scan costs a tuple id resolution and a descent a row ($heap_total - $index_total, $bitmap_rows rows)" +); + +# A keyless table's secondary index descends the table's tree by the row's +# hash, a descent a row as a keyed table's. +my ($keyless_leaves, $keyless_height) = shape('keyless'); +my ($keyless_only, $keyless_rows) = + planned('SELECT v FROM keyless WHERE v < 1000', $index_only); +my ($keyless_scan) = + planned('SELECT v FROM keyless WHERE v < 1000', $index_scan); +ok( close_to( + $keyless_scan - $keyless_only, + $keyless_rows * $keyless_height * $cpu_operator), + "a keyless table's secondary index scan costs a descent by the row's hash a row" +); + +# An index-only scan reads no row of the table, so a VACUUM, which heap's +# visibility map would change it by, leaves its cost as it was. +$node->safe_psql('postgres', 'VACUUM costed'); +my ($vacuumed) = planned('SELECT v FROM costed WHERE v < 1000', $index_only); +ok( close_to($vacuumed, $only), + "an index-only scan's cost is the same before and after a VACUUM ($only, $vacuumed)" +); + +# default_cost returns a statement's plan's total cost at default settings. +sub default_cost +{ + my ($sql) = @_; + my ($total) = + $node->safe_psql('postgres', "EXPLAIN (FORMAT JSON) $sql") =~ + /"Total Cost": ([0-9.]+)/; + return $total; +} + +# Item 16's plan choices at default settings, as the fork's own expected +# plans: before any VACUUM an index-only scan, whose table part is nothing; +# an index scan over a single-index bitmap scan, whose rows each cost a tuple +# id resolution beside the descent; and the primary key's range scan, its +# leaves the rows in order. +$node->safe_psql( + 'postgres', q{ +CREATE TABLE fresh (id int4 PRIMARY KEY, v int4, pad text); +INSERT INTO fresh SELECT g, g, repeat('z', 40) FROM generate_series(1, 20000) g; +CREATE INDEX fresh_v ON fresh (v); +ANALYZE fresh}); +my ($fresh_leaves, $fresh_height) = shape('fresh'); +is( $node->safe_psql( + 'postgres', 'EXPLAIN (COSTS OFF) SELECT v FROM fresh WHERE v < 100'), + "Index Only Scan using fresh_v on fresh\n Index Cond: (v < 100)", + 'an index-only scan is chosen before any VACUUM'); +my ($fresh_scan, $fresh_rows) = + planned('SELECT v FROM fresh WHERE v < 100', $index_scan); +ok( close_to( + default_cost('SELECT v FROM fresh WHERE v < 100'), + $fresh_scan - $fresh_rows * $fresh_height * $cpu_operator), + 'the chosen index-only scan costs the index scan less its descents'); +is( $node->safe_psql( + 'postgres', + 'EXPLAIN (COSTS OFF) SELECT pad FROM fresh WHERE v < 1000'), + "Index Scan using fresh_v on fresh\n Index Cond: (v < 1000)", + 'an index scan is chosen over a single-index bitmap scan'); +my ($fresh_index) = + planned('SELECT pad FROM fresh WHERE v < 1000', $index_scan); +my ($fresh_bitmap) = planned('SELECT pad FROM fresh WHERE v < 1000', + 'SET enable_indexscan = off'); +ok( close_to(default_cost('SELECT pad FROM fresh WHERE v < 1000'), + $fresh_index) + && $fresh_index < $fresh_bitmap, + "the chosen index scan costs less than the bitmap scan ($fresh_index, $fresh_bitmap)" +); +is( $node->safe_psql( + 'postgres', + 'EXPLAIN (COSTS OFF) SELECT * FROM fresh WHERE id BETWEEN 100 AND 300' + ), + "Index Scan using fresh_pkey on fresh\n Index Cond: ((id >= 100) AND (id <= 300))", + 'the primary key\'s range scan is chosen'); +my ($fresh_key_only) = + planned('SELECT id FROM fresh WHERE id BETWEEN 100 AND 300', $index_only); +ok( close_to( + default_cost('SELECT * FROM fresh WHERE id BETWEEN 100 AND 300'), + $fresh_key_only), + 'the chosen key range scan in the node cache costs its index-only scan\'s' +); + +# leaves_read is Mackert and Lohman's count of a tree's leaves that fetches +# read past a cache of so many leaves, as the fork's cost counts them. +sub leaves_read +{ + my ($fetches, $leaves, $cached) = @_; + return 0 if $leaves <= $cached; + my $limit = 2 * $leaves * $cached / (2 * $leaves - $cached); + return $fetches <= $limit + ? ceil(2 * $leaves * $fetches / (2 * $leaves + $fetches)) + : ceil($cached + ($fetches - $limit) * ($leaves - $cached) / $leaves); +} + +# Past the node cache, a secondary index scan also reads Mackert and Lohman's +# count of the table's leaves at random, with the node cache in +# effective_cache_size's place. +$node->append_conf('postgresql.conf', 'node_cache_size = 1024kB'); +$node->restart; +$node->safe_psql( + 'postgres', q{ +CREATE TABLE wide (id int4 PRIMARY KEY, v int4, pad text); +INSERT INTO wide SELECT g, (g * 7919) % 60000, repeat('y', 400) FROM generate_series(1, 60000) g; +CREATE INDEX wide_v ON wide (v); +ANALYZE wide}); +my ($wide_leaves, $wide_height) = shape('wide'); +my $cached = 1024 * 1024 / 8192; +ok( $wide_leaves > $cached, + "the wide table's leaves ($wide_leaves) exceed the node cache's $cached"); +my ($wide_only, $wide_rows) = + planned('SELECT v FROM wide WHERE v < 3000', $index_only); +my ($wide_scan) = planned('SELECT v FROM wide WHERE v < 3000', $index_scan); +my $read = leaves_read($wide_rows, $wide_leaves, $cached); +ok( close_to( + $wide_scan - $wide_only, + $wide_rows * $wide_height * $cpu_operator + $read * 4), + "a secondary index scan past the node cache reads Mackert and Lohman's leaves at random ($wide_scan - $wide_only, $wide_rows rows, $read leaves)" +); + +# The primary key's own index past the node cache reads its leaves in key +# order, the first at random and the rest in sequence: the selectivity's +# share of the leaves, which the plan's rows give to within a row. +my ($wide_key_only, $wide_key_rows) = + planned('SELECT id FROM wide WHERE id <= 3000', $index_only); +my ($wide_key_scan) = + planned('SELECT id FROM wide WHERE id <= 3000', $index_scan); +my $wide_tuples = $node->safe_psql('postgres', + "SELECT reltuples FROM pg_class WHERE relname = 'wide'"); +my @key_reads = map { ceil($_ / $wide_tuples * $wide_leaves) } + ($wide_key_rows - 0.5, $wide_key_rows + 0.5); +ok( ( grep { close_to($wide_key_scan - $wide_key_only, 4 + ($_ - 1)) } + @key_reads), + "the primary key's own index scan past the node cache reads its leaves in order ($wide_key_scan - $wide_key_only, reads @key_reads)" +); + +# A repeated inner index scan, a nested loop's parameterized one, costs each +# loop its share of Mackert and Lohman's leaves over every loop's fetches: a +# secondary index's rows and the primary key's one leaf a probe. Enough +# probes pass the node cache's share of the leaves that the count differs +# from heap's over effective_cache_size. +$node->safe_psql('postgres', + 'CREATE TABLE probes (x int4); INSERT INTO probes SELECT g * 13 FROM generate_series(1, 4000) g; ANALYZE probes' +); +my $nested = + 'SET enable_hashjoin = off; SET enable_mergejoin = off; SET enable_material = off; SET enable_memoize = off; SET enable_bitmapscan = off'; + +# inner returns a nested loop's inner index scan's cost a loop, its rows a +# loop, and the loops, its outer scan's rows. +sub inner +{ + my ($sql, $settings) = @_; + my $json = $node->safe_psql('postgres', + "$nested; $settings; EXPLAIN (FORMAT JSON) $sql"); + my ($loops) = $json =~ /"Node Type": "Seq Scan".*?"Plan Rows": ([0-9]+)/s; + my ($cost, $rows) = $json =~ + /"Node Type": "Index (?:Only )?Scan".*?"Total Cost": ([0-9.]+),\s*"Plan Rows": ([0-9]+)/s; + return ($cost, $rows, $loops); +} +my ($loop_only, $loop_rows, $loops) = + inner('SELECT w.v FROM probes p JOIN wide w ON w.v = p.x', ''); +my ($loop_scan) = inner('SELECT w.pad FROM probes p JOIN wide w ON w.v = p.x', + 'SET enable_indexonlyscan = off'); +ok( close_to( + $loop_scan - $loop_only, + $loop_rows * + $wide_height * + $cpu_operator + + leaves_read($loop_rows * $loops, $wide_leaves, $cached) * 4 / $loops + ), + "a repeated secondary index scan costs its loop's share of the leaves read ($loop_scan - $loop_only, $loops loops)" +); +my ($key_loop_only) = + inner('SELECT w.id FROM probes p JOIN wide w ON w.id = p.x', ''); +my ($key_loop_scan) = + inner('SELECT w.pad FROM probes p JOIN wide w ON w.id = p.x', + 'SET enable_indexonlyscan = off'); +ok( close_to( + $key_loop_scan - $key_loop_only, + leaves_read($loops, $wide_leaves, $cached) * 4 / $loops), + "a repeated primary key index scan costs its loop's share of the leaves read ($key_loop_scan - $key_loop_only, $loops loops)" +); + +# A lossy bitmap: with 64kB of work_mem the bitmap holds 1024 entries, the +# size of a page's entry and two pointers, so of the pages compute_bitmap_pages +# fetches all but half the entries are lossy, and each costs a range scan of +# the tuple id tree: a descent, and its leaf read at random past the node +# cache. A scan of every row keeps the rows fetched the same either way. +my ($wide_tid_leaves, $wide_tid_height) = (shape('wide'))[ 2, 3 ]; +my $lossy_sql = 'SELECT pad FROM wide WHERE v >= 0'; +my ($lossy_small) = + planned($lossy_sql, "SET enable_indexscan = off; SET work_mem = '64kB'"); +my ($lossy_large) = + planned($lossy_sql, "SET enable_indexscan = off; SET work_mem = '64MB'"); +my $wide_pages = $node->safe_psql('postgres', + "SELECT relpages FROM pg_class WHERE relname = 'wide'"); +my $fetched = + 2 * $wide_pages * $wide_tuples / (2 * $wide_pages + $wide_tuples); +$fetched = $fetched >= $wide_pages ? $wide_pages : ceil($fetched); +my $lossy = $fetched - 1024 / 2; +ok( $lossy > 0 && close_to( + $lossy_small - $lossy_large, + $lossy * ( + $wide_tid_height * $cpu_operator + + ($wide_tid_leaves > $cached ? 4 : 0))), + "a lossy bitmap's blocks each cost a tuple id range scan ($lossy_small - $lossy_large, $lossy lossy blocks)" +); + +$node->stop; +done_testing(); diff --git a/src/doltgres/test/t/132_heap_only_contrib.pl b/src/doltgres/test/t/132_heap_only_contrib.pl new file mode 100644 index 0000000000000..8ac8a5a963dee --- /dev/null +++ b/src/doltgres/test/t/132_heap_only_contrib.pl @@ -0,0 +1,175 @@ +# Copyright 2026 Dolthub, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The heap-only contrib modules refuse the fork's relations (design item 5; +# bugs row 330): each function of pageinspect, pg_surgery, +# pgstattuple, and amcheck that reads a heap's or an index's pages refuses a +# relation the fork stores with its own access method or relation kind +# check's SQLSTATE and message, as the reference raises them for a table of +# heap2 (heap's handler under another name), an index of another method, or +# a view, while an index of a method the fork doesn't store reads as +# Postgres' own; and pg_prewarm, whose reading of the fork's trees is bugs row +# 331's, returns without failing the server. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('contrib'); +$node->init; +$node->start; +$node->safe_psql( + 'postgres', q{ +CREATE EXTENSION pageinspect; +CREATE EXTENSION pg_visibility; +CREATE EXTENSION pg_surgery; +CREATE EXTENSION pgstattuple; +CREATE EXTENSION amcheck; +CREATE EXTENSION pg_prewarm; +CREATE TABLE forked (id int4 PRIMARY KEY, v int4); +INSERT INTO forked SELECT g, g FROM generate_series(1, 100) g; +CREATE INDEX forked_v ON forked (v); +CREATE INDEX forked_hash ON forked USING hash (v)}); + +# refused returns a statement's error as its SQLSTATE and message. +sub refused +{ + my ($sql) = @_; + my ($ret, $out, $err) = + $node->psql('postgres', "\\set VERBOSITY verbose\n$sql"); + my ($error) = $err =~ /ERROR: ([0-9A-Z]{5}: [^\n]*)/; + return $error // "no error: $out"; +} + +# Each case: the call on a fork relation, and the reference's error for the +# same check, with the fork relation's name. +my @cases = ( + [ + q{SELECT tuple_data_split('forked'::regclass, '\x00'::bytea, 0, 1, NULL)}, + '0A000: only heap AM is supported' + ], + [ + q{SELECT get_raw_page('forked', 0)}, + '42809: cannot get raw page from relation "forked"' + ], + [ + q{SELECT bt_page_items('forked_v', 1)}, + '42809: "forked_v" is not a btree index' + ], + [ + q{SELECT bt_metap('forked_v')}, + '42809: "forked_v" is not a btree index' + ], + [ + q{SELECT heap_force_kill('forked', ARRAY['(0,1)']::tid[])}, + '0A000: only heap AM is supported' + ], + [ q{SELECT pgstattuple('forked')}, '0A000: only heap AM is supported' ], + [ + q{SELECT pgstattuple('forked_v')}, + '0A000: index "forked_v" (unknown index) is not supported' + ], + [ + q{SELECT pgstattuple_approx('forked')}, + '0A000: only heap AM is supported' + ], + [ + q{SELECT pgstatindex('forked_v')}, + '42809: relation "forked_v" is not a btree index' + ], + [ q{SELECT verify_heapam('forked')}, '0A000: only heap AM is supported' ], + [ + q{SELECT bt_index_check('forked_v')}, + '0A000: expected "btree" index as targets for verification' + ],); +for my $case (@cases) +{ + my ($sql, $wanted) = @$case; + is(refused($sql), $wanted, "$sql refuses a fork relation"); +} + +# An index of a method whose entries the fork doesn't store yet, hash here, +# keeps Postgres' own pages, which the modules read as they read heap's: the +# hash, GiST, BRIN, and GIN functions' checks refuse such an index once the +# fork stores its method's entries, each method in its own piece after the +# switch (PLAN's Order 2d). +like( + $node->safe_psql('postgres', q{SELECT pgstathashindex('forked_hash')}), + qr/^\([0-9]+,/, + "an index of a method the fork doesn't store reads as Postgres' own"); + +# get_raw_page refuses a fork table's main fork alone: its visibility map +# fork, which the fork never writes, has no blocks, as the reference's error +# for a fork with none says. Its free space map fork's page, Postgres' own +# (bugs row 87), is bugs row 333's case. +is( refused(q{SELECT get_raw_page('forked', 'vm', 0)}), + '22023: block number 0 is out of range for relation "forked"', + 'a fork table\'s visibility map fork has no blocks'); + +# pg_prewarm doesn't yet read a fork relation's trees (bugs row 331), but its +# call returns, whatever it answers, and the server stays up. +my ($ret) = $node->psql('postgres', q{SELECT pg_prewarm('forked')}); +ok(defined $ret, 'pg_prewarm of a fork table returns'); +is($node->safe_psql('postgres', 'SELECT count(*) FROM forked'), + '100', 'the server reads the table after pg_prewarm'); + +# Bugs row 348: pg_visibility works on a fork table, which keeps no +# visibility map since every block is all-visible and all-frozen (design +# item 5): each block within the table reads all-visible and all-frozen, in +# the map and on its page, and a block past it neither; the summary counts +# every block; pg_check_visible and pg_check_frozen find no tuple against +# that, a running insert's and a deleted row's included; and truncating the +# map changes none of it. +$node->safe_psql('postgres', + q{CREATE TABLE visible (k int4, v text); INSERT INTO visible SELECT g, repeat('v', 200) FROM generate_series(1, 2000) g; DELETE FROM visible WHERE k <= 10} +); +my $running = $node->background_psql('postgres'); +$running->query_safe( + q{BEGIN; INSERT INTO visible SELECT g, 'running' FROM generate_series(2001, 2100) g} +); +my $mapped = q{SELECT count(*) || ' ' || count(*) FILTER (WHERE all_visible AND all_frozen) FROM pg_visibility_map('visible')}; +my $blocks = $node->safe_psql('postgres', q{SELECT count(*) FROM pg_visibility_map('visible')}); +ok($blocks > 1, 'a fork table of 2,000 rows maps more than one block'); +is($node->safe_psql('postgres', $mapped), + "$blocks $blocks", 'every block of a fork table is all-visible and all-frozen in its map'); +is( $node->safe_psql( + 'postgres', + q{SELECT count(*) FILTER (WHERE all_visible AND all_frozen AND pd_all_visible) FROM pg_visibility('visible')} + ), + $blocks, + 'and on its page'); +is( $node->safe_psql( + 'postgres', + q{SELECT pg_visibility_map('visible', 0) || ' ' || pg_visibility('visible', 0) || ' ' || pg_visibility_map('visible', 1000000) || ' ' || pg_visibility('visible', 1000000)} + ), + '(t,t) (t,t,t) (f,f) (f,f,f)', + 'a block of the table reads all-visible and all-frozen, and a block past it neither'); +is( $node->safe_psql('postgres', q{SELECT pg_visibility_map_summary('visible')}), + "($blocks,$blocks)", 'the summary counts every block both'); +is( $node->safe_psql( + 'postgres', + q{SELECT (SELECT count(*) FROM pg_check_visible('visible')) || ' ' || (SELECT count(*) FROM pg_check_frozen('visible'))} + ), + '0 0', + 'no tuple of a fork table contradicts its blocks, with a running insert and a deleted row'); +$running->query_safe('ROLLBACK'); +$running->quit; +$node->safe_psql('postgres', q{SELECT pg_truncate_visibility_map('visible')}); +is($node->safe_psql('postgres', $mapped), + "$blocks $blocks", 'truncating the map leaves every block all-visible and all-frozen'); + +$node->stop; +done_testing(); diff --git a/src/doltgres/test/t/133_identity_spill.pl b/src/doltgres/test/t/133_identity_spill.pl new file mode 100644 index 0000000000000..aedcc9ede3398 --- /dev/null +++ b/src/doltgres/test/t/133_identity_spill.pl @@ -0,0 +1,218 @@ +# Copyright 2026 Dolthub, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Bugs row 291: a transaction's own identity tree versions spill past the +# spill threshold, as its other trees' do. At delta_memory 1MB a transaction +# making hundreds of tables spills its identity pairs, reads them back in its +# own transaction and, once it commits, in another; an abort leaves no pair +# and its files go; a restart after a checkpoint that listed a running +# transaction's identity spill finds every pair it committed; and a 1MB +# standby replays a transaction of a thousand tables. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $settings = + "delta_memory = 1MB\nfold_threshold = 1GB\nfold_timeout = 3600s\ncheckpoint_timeout = 1h"; +my $node = PostgreSQL::Test::Cluster->new('identity'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', $settings); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION doltgres_test'); +$node->backup('replayed'); +my $standby = PostgreSQL::Test::Cluster->new('replayed'); +$standby->init_from_backup($node, 'replayed', has_streaming => 1); +$standby->append_conf('postgresql.conf', $settings); +$standby->start; + +# made returns a DO block that makes count tables of a prefix, a key and a +# text column each. +sub made +{ + my ($prefix, $count) = @_; + return + "DO \$\$ BEGIN FOR i IN 1 .. $count LOOP EXECUTE format('CREATE TABLE ${prefix}_%s (k int4, v text)', i); END LOOP; END \$\$"; +} + +# identity_files returns the count of a transaction's identity spill files. +sub identity_files +{ + my ($xid) = @_; + return $node->safe_psql('postgres', + "SELECT count(*) FROM pg_ls_dir('pg_versioned/' || (SELECT oid FROM pg_database WHERE datname = 'postgres') || '/delta/spill', true, false) f WHERE f LIKE 'identity-$xid-%'" + ); +} + +# A transaction's spilled pairs read back in it and, after its commit, in +# another session. +my $own = $node->background_psql('postgres'); +$own->query_safe("SET statement_timeout = '300s'"); +my $owner = $own->query_safe('BEGIN; SELECT txid_current()'); +$own->query_safe(made('own', 400)); +cmp_ok(identity_files($owner), '>', 0, + "a transaction making 400 tables at 1MB spills its identity pairs"); +my $first = $own->query_safe("SELECT 'own_1'::regclass::oid"); +my $found = $own->query_safe("SELECT doltgres_test_identity(2, $first, 0)"); +isnt($found, 'none', "its own transaction finds a spilled table's pair"); +$own->query_safe('INSERT INTO own_1 VALUES (1, \'a\')'); +is($own->query_safe('SELECT count(*) FROM own_1'), + '1', "and writes and reads that table"); +$own->query_safe('COMMIT'); +$own->quit; +is( $node->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $first, 0)"), + $found, + "after its commit another session finds the same pair"); +is($node->safe_psql('postgres', 'SELECT count(*) FROM own_1'), + '1', "and reads the table's row"); + +# An abort after a spill leaves no pair, and a later checkpoint removes its +# files. +my $abort = $node->background_psql('postgres'); +$abort->query_safe("SET statement_timeout = '300s'"); +my $aborted = $abort->query_safe('BEGIN; SELECT txid_current()'); +$abort->query_safe(made('aborted', 400)); +my $gone = $abort->query_safe("SELECT 'aborted_1'::regclass::oid"); +cmp_ok(identity_files($aborted), + '>', 0, "an aborting transaction's identity pairs spill"); +$abort->query_safe('ROLLBACK'); +$abort->quit; +is( $node->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $gone, 0)"), + 'none', + "after its abort no pair names its table"); +$node->safe_psql('postgres', 'CHECKPOINT') for 1 .. 3; +is(identity_files($aborted), + '0', "and its identity spill files are gone after three checkpoints"); +my ($code, $out, $err) = $node->psql('postgres', + "SELECT count(*) FROM pg_class WHERE relname LIKE 'own\\_%' AND doltgres_test_identity(2, oid, 0) <> 'none'" +); +is( $code . $err . ',' . $out, + '0,400', + "and once those checkpoints' saves fold the first transaction's spill, every table it made has its pair" +); + +# A checkpoint lists a running transaction's identity spill; after its commit +# and a crash, the restart finds every pair from the listed file. +my $listed = $node->background_psql('postgres'); +$listed->query_safe("SET statement_timeout = '300s'"); +my $lister = $listed->query_safe('BEGIN; SELECT txid_current()'); +$listed->query_safe(made('listed', 400)); +cmp_ok(identity_files($lister), '>', 0, + "a third transaction's identity pairs spill"); +my $kept = $listed->query_safe("SELECT 'listed_1'::regclass::oid"); +my $kept_id = + $listed->query_safe("SELECT doltgres_test_identity(2, $kept, 0)"); +$node->safe_psql('postgres', 'CHECKPOINT'); +$listed->query_safe('COMMIT'); +$listed->quit; +$node->stop('immediate'); +$node->start; +is( $node->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $kept, 0)"), + $kept_id, + "after a crash the restart finds a pair the checkpoint's listed spill holds" +); +is( $node->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_class WHERE relname LIKE 'listed\\_%' AND doltgres_test_identity(2, oid, 0) <> 'none'" + ), + '400', + "and every table the transaction made has its pair"); +$node->safe_psql('postgres', 'CHECKPOINT'); +is( $node->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $kept, 0)"), + $kept_id, + "and still does once a checkpoint's save folds them"); + +for my $round (1 .. 3) +{ + $node->safe_psql('postgres', + "INSERT INTO own_1 VALUES ($round + 1, 'b'); CHECKPOINT"); +} +is(identity_files($lister), '0', + "and the save's folded identity spill files are gone two checkpoints later" +); + +# A 1MB standby replays one transaction of a thousand tables. +$node->safe_psql('postgres', + "SET statement_timeout = '300s'; BEGIN; " + . made('standby', 1000) + . "; COMMIT;"); +$node->wait_for_replay_catchup($standby); +my $pairs = + q{SELECT string_agg(relname || '=' || doltgres_test_identity(2, oid, 0), ' ' ORDER BY relname) FROM pg_class WHERE relname LIKE 'standby\_%'}; +is( $standby->safe_psql('postgres', $pairs), + $node->safe_psql('postgres', $pairs), + "a 1MB standby replays a transaction of a thousand tables, each pair as its primary's" +); + +# Another transaction's lookup of a running transaction's spilled pair waits +# for it, as it waits for an unspilled one, and then reads the pair. +my $holder = $node->background_psql('postgres'); +$holder->query_safe("SET statement_timeout = '300s'"); +my $holding = $holder->query_safe('BEGIN; SELECT txid_current()'); +$holder->query_safe(made('held', 400)); +cmp_ok(identity_files($holding), + '>', 0, "a running transaction's identity pairs spill"); +my $held = $holder->query_safe("SELECT 'held_1'::regclass::oid"); +my $held_id = + $holder->query_safe("SELECT doltgres_test_identity(2, $held, 0)"); +my (undef, undef, $waited) = $node->psql('postgres', + "SET statement_timeout = '2s'; SELECT doltgres_test_identity(2, $held, 0)" +); +like( + $waited, + qr/canceling statement due to statement timeout/, + "another transaction's lookup of its spilled pair waits for it"); +$holder->query_safe('COMMIT'); +$holder->quit; +is( $node->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $held, 0)"), + $held_id, + "and reads the pair once it commits"); + +# A 1MB standby that restarts at a restartpoint, while a running transaction's +# replayed identity versions sit in its listed spills on both sides of the +# restartpoint's redo point, pushes none of them again as it replays. +my $replaying = $node->background_psql('postgres'); +$replaying->query_safe("SET statement_timeout = '300s'"); +$replaying->query_safe('BEGIN; SELECT txid_current()'); +$replaying->query_safe(made('early', 400)); +$node->safe_psql('postgres', 'CHECKPOINT'); +$replaying->query_safe(made('late', 400)); +my $oids = $replaying->query_safe( + "SELECT string_agg(oid::text, ',') FROM pg_class WHERE relname LIKE 'early\\_%' OR relname LIKE 'late\\_%'" +); +$node->wait_for_replay_catchup($standby); +$standby->safe_psql('postgres', 'CHECKPOINT'); +$standby->stop('immediate'); +$standby->start; +$node->wait_for_replay_catchup($standby); +is( $standby->safe_psql( + 'postgres', + "SELECT count(*) FROM unnest('{$oids}'::oid[]) o WHERE doltgres_test_identity_versions(2, o) <> '1'" + ), + '0', + "a restarted 1MB standby's redo pushes no version its listed spills hold" +); +$replaying->query_safe('COMMIT'); +$replaying->quit; + +$standby->stop; +$node->stop; +done_testing(); diff --git a/src/doltgres/test/t/134_saves.pl b/src/doltgres/test/t/134_saves.pl new file mode 100644 index 0000000000000..9b2f2a0adafc5 --- /dev/null +++ b/src/doltgres/test/t/134_saves.pl @@ -0,0 +1,396 @@ +# Copyright 2026 Dolthub, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Bugs row 340: item 6's saves outside a checkpoint. At delta_memory 1MB, +# committed transactions' identity versions, which only a save frees, don't +# stop later writers: a refused writer runs the save for them, so many +# committed CREATE TABLEs finish with no checkpoint; a states' pool at its +# smallest, filled with unsaved states, makes room by a save with no +# checkpoint; and crash recovery replays a fold exactly. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('saves'); +$node->init; +$node->append_conf('postgresql.conf', + "delta_memory = 1MB\nfold_threshold = 1GB\nfold_timeout = 3600s\ncheckpoint_timeout = 1h" +); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION doltgres_test'); + +my $checkpoints = + 'SELECT num_timed + num_requested FROM pg_stat_checkpointer'; + +# Many committed CREATE TABLEs, each its own transaction, finish at 1MB with +# no checkpoint: the writers the budget refuses save the committed identity +# versions. +my $before = $node->safe_psql('postgres', $checkpoints); +my ($code, $out, $err) = $node->psql('postgres', + "SET statement_timeout = '300s'; DO \$\$ BEGIN FOR i IN 1 .. 2000 LOOP EXECUTE format('CREATE TABLE committed_%s (k int4, v text)', i); COMMIT; END LOOP; END \$\$" +); +is($code . $err, + '0', + "2,000 committed CREATE TABLEs at 1MB finish with no checkpoint run"); +is($node->safe_psql('postgres', $checkpoints), + $before, "and no checkpoint ran meanwhile"); + +$node->stop; + +# A states' pool at its smallest, filled with storages' unsaved states, makes +# room for another by saving with no checkpoint. +my $pooled = PostgreSQL::Test::Cluster->new('pooled'); +$pooled->init; +$pooled->append_conf('postgresql.conf', + "max_connections = 10\nmax_locks_per_transaction = 10\nfold_threshold = 1GB\nfold_timeout = 3600s\ncheckpoint_timeout = 1h" +); +$pooled->start; +$pooled->safe_psql('postgres', 'CREATE EXTENSION doltgres_test'); + +# Coverage of a restart after a non-checkpoint save (no failing-before: redo +# replays every identity change after the last checkpoint's redo point, so a +# run's older version never outlives a newer one): a table dropped and made +# again under its name, whose id pairs with its new OID, and a table made and +# dropped, which no pair names, both before the saves the pool's makes run. +$pooled->safe_psql('postgres', 'CREATE TABLE changed (k int4); CHECKPOINT'); +my $old = $pooled->safe_psql('postgres', "SELECT 'changed'::regclass::oid"); +my $id = + $pooled->safe_psql('postgres', "SELECT doltgres_test_identity(2, $old, 0)"); +$pooled->safe_psql('postgres', 'DROP TABLE changed'); +$pooled->safe_psql('postgres', 'CREATE TABLE changed (k int4)'); +my $new = $pooled->safe_psql('postgres', "SELECT 'changed'::regclass::oid"); +$pooled->safe_psql('postgres', 'CREATE TABLE gone (k int4)'); +my $gone = $pooled->safe_psql('postgres', "SELECT 'gone'::regclass::oid"); +$pooled->safe_psql('postgres', 'DROP TABLE gone'); +my $slots = + $pooled->safe_psql('postgres', 'SELECT doltgres_test_relation_locks()'); +my $pool_before = $pooled->safe_psql('postgres', $checkpoints); +my $made = $slots + 50; +($code, $out, $err) = $pooled->psql('postgres', + "SET statement_timeout = '300s'; DO \$\$ BEGIN FOR i IN 1 .. $made LOOP EXECUTE format('CREATE TABLE pooled_%s (k int4)', i); EXECUTE format('INSERT INTO pooled_%s VALUES (1)', i); COMMIT; END LOOP; END \$\$" +); +is($code . $err, + '0', "writing into more storages than the states' pool holds finishes"); +is($pooled->safe_psql('postgres', $checkpoints), + $pool_before, "and no checkpoint ran meanwhile"); +is( $pooled->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_class WHERE relname LIKE 'pooled\\_%' AND relkind = 'r'" + ), + $made, + "and every table it made is there"); +$pooled->stop('immediate'); +$pooled->start; +is( $pooled->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $new, 0)"), + $id, + "after a crash past the pool's saves, a table made again under its name pairs its id with its new OID" +); +is( $pooled->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $old, 0)"), + 'none', + "and its old OID with nothing"); +is( $pooled->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $gone, 0)"), + 'none', + "and a table made and dropped before the saves has no pair"); +$pooled->stop; + +# Crash recovery replays a fold after its rows' commit records exactly: the +# replayed fold takes out the committed row version the live fold took out, +# since a transaction whose commit redo replayed reads as committed there. +my $replayed = PostgreSQL::Test::Cluster->new('replayed'); +$replayed->init; +$replayed->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\ncheckpoint_timeout = 1h"); +$replayed->start; +$replayed->safe_psql('postgres', 'CREATE EXTENSION doltgres_test'); +$replayed->safe_psql('postgres', + 'CREATE TABLE folded (k int4 PRIMARY KEY, v text); CHECKPOINT'); +$replayed->safe_psql('postgres', "INSERT INTO folded VALUES (1, 'a')"); +$replayed->safe_psql('postgres', "SELECT doltgres_test_fold('folded')"); +is( $replayed->safe_psql( + 'postgres', "SELECT doltgres_test_held('folded', '(0,1)')"), + 'f', + "a fold takes out a committed row's version"); +$replayed->safe_psql('postgres', 'SELECT pg_switch_wal()'); +$replayed->stop('immediate'); +$replayed->start; +is( $replayed->safe_psql( + 'postgres', "SELECT doltgres_test_held('folded', '(0,1)')"), + 'f', + "and after a crash its replay after the row's commit record takes it out too" +); +is($replayed->safe_psql('postgres', 'SELECT v FROM folded WHERE k = 1'), + 'a', "and the row reads back from the base"); +$replayed->stop; + +# Bugs row 350: a durable manifest names only what is durable. A save waits +# for the WAL through the record of each base it gathers, lists every pack +# whose add its repository defers, and waits for the WAL through its fold +# point when it folds an identity pair. The walwriter waits ten seconds +# between flushes, so nothing but the save flushes the WAL in these cases. +my $logged = PostgreSQL::Test::Cluster->new('logged'); +$logged->init; +$logged->append_conf('postgresql.conf', + "autovacuum = off\nfold_threshold = 1GB\nfold_timeout = 3600s\ncheckpoint_timeout = 1h\nwal_writer_delay = 10s\nrestart_after_crash = on" +); +$logged->start; +$logged->safe_psql('postgres', + 'CREATE EXTENSION doltgres_test; CREATE EXTENSION injection_points; CREATE EXTENSION pg_walinspect' +); + +# landed runs a save in a session of its own, paused once its manifest is +# written, and returns the session. +my $landed = sub { + my $saver = $logged->background_psql('postgres', on_error_stop => 0); + $saver->query_safe('SELECT injection_points_set_local()'); + $saver->query_safe( + "SELECT injection_points_attach('doltgres-repo-save-landed', 'wait')" + ); + $saver->query_until(qr/start/, + "\\echo start\nSELECT doltgres_test_save();\n"); + $logged->wait_for_event('client backend', 'doltgres-repo-save-landed'); + return $saver; +}; + +# A fold paused after its install, a save that gathers its base, and a crash: +# the save flushed the fold's record, so the restart reads the row once. The +# tables of the row-once cases have no key, so a row both a base and the delta +# hold reads twice. +$logged->safe_psql('postgres', + 'CREATE TABLE logged_fold (k int4, v text); CHECKPOINT'); +my $start = + $logged->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$logged->safe_psql('postgres', "INSERT INTO logged_fold VALUES (1, 'a')"); +my $folder = $logged->background_psql('postgres', on_error_stop => 0); +$folder->query_safe('SELECT injection_points_set_local()'); +$folder->query_safe( + "SELECT injection_points_attach('doltgres-fold-installed', 'wait')"); +$folder->query_until(qr/start/, + "\\echo start\nSELECT doltgres_test_fold('logged_fold');\n"); +$logged->wait_for_event('client backend', 'doltgres-fold-installed'); +my $saver = $landed->(); +is( $logged->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_get_wal_records_info('$start', pg_current_wal_flush_lsn()) WHERE record_type = 'BaseAdvanced'" + ), + '1', + "a save's manifest names a fold's base only once the base's record is flushed" +); +$logged->stop('immediate'); +eval { $folder->quit }; +eval { $saver->quit }; +$logged->start; +is($logged->safe_psql('postgres', 'SELECT count(*) FROM logged_fold'), + '1', "and after a crash the restart reads the folded row once"); +is( $logged->safe_psql( + 'postgres', "SELECT doltgres_test_held('logged_fold', '(0,1)')"), + 'f', + "and the replay of the fold's record takes its version out of the delta"); + +# A save whose own base tree's pack has a deferred add, and a crash: the save +# listed the pack before its manifest named it, so the restart finds every +# root the manifest names. +$logged->safe_psql('postgres', + "CREATE TABLE deferred_pack (k int4 PRIMARY KEY, v text); INSERT INTO deferred_pack SELECT g, 'd' || g FROM generate_series(1, 100) g" +); +$logged->safe_psql('postgres', "SELECT doltgres_test_fold('deferred_pack')"); +$saver = $landed->(); +$logged->stop('immediate'); +eval { $saver->quit }; +$logged->start; +is( $logged->safe_psql('postgres', 'SELECT count(*) FROM deferred_pack'), + '100', + "after a crash past a save, the restart finds every root its manifest names" +); + +# A table made with an asynchronous commit, a save that folds its identity +# pair, and a crash: the save flushed the commit's record, so the restart has +# the table with its pair. +my $async = $logged->background_psql('postgres'); +$async->query_safe( + "SET synchronous_commit = off; CREATE TABLE async_made (k int4)"); +my $made_oid = + $logged->safe_psql('postgres', "SELECT 'async_made'::regclass::oid"); +my $made_id = $logged->safe_psql('postgres', + "SELECT doltgres_test_identity(2, $made_oid, 0)"); +$saver = $landed->(); +$logged->stop('immediate'); +eval { $async->quit }; +eval { $saver->quit }; +$logged->start; +is( $logged->safe_psql( + 'postgres', "SELECT count(*) FROM pg_class WHERE oid = $made_oid"), + '1', + "after a crash past a save that folded an asynchronous commit's identity pair, the table is there" +); +is( $logged->safe_psql( + 'postgres', "SELECT doltgres_test_identity(2, $made_oid, 0)"), + $made_id, + "with its pair"); + +# A save past its fold point and a fold paused between its record's insertion +# and its base's publish (a fold point waits out the install's checkpoint +# delay, so only a save that took its point first gathers in the window): a +# reader reads the row once across the save and the publish, and the save +# names the base the fold replaces, so after a crash the restart reads each +# row once. +my $pointed = sub { + my $saver = $logged->background_psql('postgres', on_error_stop => 0); + $saver->query_safe('SELECT injection_points_set_local()'); + $saver->query_safe( + "SELECT injection_points_attach('doltgres-runs-save-pointed', 'wait')" + ); + $saver->query_until(qr/start/, + "\\echo start\nSELECT doltgres_test_save();\n"); + $logged->wait_for_event('client backend', 'doltgres-runs-save-pointed'); + return $saver; +}; +my $logging = sub { + my ($table) = @_; + my $logger = $logged->background_psql('postgres', on_error_stop => 0); + $logger->query_safe('SELECT injection_points_set_local()'); + $logger->query_safe( + "SELECT injection_points_attach('doltgres-state-logging', 'wait')"); + $logger->query_until(qr/start/, + "\\echo start\nSELECT doltgres_test_fold('$table');\n"); + $logged->wait_for_event('client backend', 'doltgres-state-logging'); + return $logger; +}; +my $saved = sub { + my ($saver) = @_; + $logged->safe_psql('postgres', + "SELECT injection_points_wakeup('doltgres-runs-save-pointed')"); + $logged->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_stat_activity WHERE query LIKE 'SELECT doltgres_test_save()%' AND state = 'active'" + ) or die "the save never finished"; + $saver->quit; +}; +$logged->safe_psql('postgres', + 'CREATE TABLE logging_fold (k int4, v text); CHECKPOINT'); +$logged->safe_psql('postgres', "INSERT INTO logging_fold VALUES (1, 'p')"); +$saver = $pointed->(); +$folder = $logging->('logging_fold'); +my $reader = $logged->background_psql('postgres'); +is( $reader->query_safe( + 'BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM logging_fold' + ), + '1', + "a reader while a fold's record is in and its base unpublished reads the row once" +); +$saved->($saver); +$logged->safe_psql('postgres', + "SELECT injection_points_wakeup('doltgres-state-logging')"); +$folder->quit; +is($reader->query_safe('SELECT count(*) FROM logging_fold'), + '1', "and still once after a save in the window and the base's publish"); +$reader->query_safe('COMMIT'); +$reader->quit; +my $second = $logged->safe_psql('postgres', + "SELECT pg_current_wal_insert_lsn() - '0/0'::pg_lsn"); +$logged->safe_psql('postgres', "INSERT INTO logging_fold VALUES (2, 'q')"); +$saver = $pointed->(); +$folder = $logging->('logging_fold'); +$saved->($saver); +$logged->stop('immediate'); +eval { $folder->quit }; +$logged->start; +my (undef, $folded) = split / /, + $logged->safe_psql('postgres', + "SELECT doltgres_test_base_points('logging_fold')"); +is( ($folded < $second ? 'below' : 'past') . ' ' + . $logged->safe_psql( + 'postgres', "SELECT doltgres_test_held('logging_fold', '(0,2)')"), + 'below t', + "after a crash past a save that gathered before a base was published, the base is the one the save named and the delta holds the later row" +); +is($logged->safe_psql('postgres', 'SELECT count(*) FROM logging_fold'), + '2', "and the restart reads each row once"); + +# An error inside a logging install's window stops the server, and recovery +# replays the records before it, so the row reads once. +$logged->safe_psql('postgres', + "CREATE TABLE stopped_fold (k int4, v text); INSERT INTO stopped_fold VALUES (1, 's')" +); +$folder = $logged->background_psql('postgres', on_error_stop => 0); +$folder->query_safe('SELECT injection_points_set_local()'); +$folder->query_safe( + "SELECT injection_points_attach('doltgres-state-logging', 'error')"); +eval { $folder->query("SELECT doltgres_test_fold('stopped_fold')") }; +eval { $folder->quit }; +ok( $logged->log_contains( + 'PANIC: error triggered for injection point doltgres-state-logging'), + "an error inside a logging install's window stops the server"); +$logged->poll_query_until('postgres', 'SELECT true') + or die "the server never came back after the stop"; +is($logged->safe_psql('postgres', 'SELECT count(*) FROM stopped_fold'), + '1', "and recovery reads the row once"); + +# Bugs row 354: no record a process logs from a base's content lands before +# the base's own record, since a base is published only after its record is +# in. VACUUM reading the previous base while a fold's record is in and its +# base unpublished logs its frees after the fold's record, and a standby that +# replays both reads every row at the tuple id its primary does. +my $vacuuming = PostgreSQL::Test::Cluster->new('vacuuming'); +$vacuuming->init(allows_streaming => 1); +$vacuuming->append_conf('postgresql.conf', + "autovacuum = off\nfold_threshold = 1GB\nfold_timeout = 3600s\ncheckpoint_timeout = 1h" +); +$vacuuming->start; +$vacuuming->safe_psql('postgres', + 'CREATE EXTENSION doltgres_test; CREATE EXTENSION injection_points'); +$vacuuming->backup('vacuuming_backup'); +my $vacuumed = PostgreSQL::Test::Cluster->new('vacuumed'); +$vacuumed->init_from_backup($vacuuming, 'vacuuming_backup', + has_streaming => 1); +$vacuumed->start; +$vacuuming->safe_psql('postgres', + "CREATE TABLE vacuumed (k int4, v text); INSERT INTO vacuumed SELECT g, 'v' || g FROM generate_series(1, 20) g" +); +$vacuuming->safe_psql('postgres', "SELECT doltgres_test_fold('vacuumed')"); +$vacuuming->safe_psql('postgres', 'DELETE FROM vacuumed WHERE k <= 5'); +$vacuuming->safe_psql('postgres', "SELECT doltgres_test_fold('vacuumed')"); +$vacuuming->safe_psql('postgres', "INSERT INTO vacuumed VALUES (21, 'w')"); +my $windowed = $vacuuming->background_psql('postgres', on_error_stop => 0); +$windowed->query_safe('SELECT injection_points_set_local()'); +$windowed->query_safe( + "SELECT injection_points_attach('doltgres-state-logging', 'wait')"); +$windowed->query_until(qr/start/, + "\\echo start\nSELECT doltgres_test_fold('vacuumed');\n"); +$vacuuming->wait_for_event('client backend', 'doltgres-state-logging'); +$vacuuming->safe_psql('postgres', + "SELECT doltgres_test_vacuum('vacuumed', 'on')"); +$vacuuming->safe_psql('postgres', + "SELECT injection_points_wakeup('doltgres-state-logging')"); +$windowed->quit; +$vacuuming->safe_psql('postgres', + "INSERT INTO vacuumed SELECT g, 'x' || g FROM generate_series(22, 30) g"); +$vacuuming->safe_psql('postgres', "SELECT doltgres_test_fold('vacuumed')"); +$vacuuming->wait_for_replay_catchup($vacuumed); +my $placed = + "SELECT string_agg(ctid::text || ':' || k, ' ' ORDER BY ctid) FROM vacuumed"; +is( $vacuumed->safe_psql('postgres', $placed), + $vacuuming->safe_psql('postgres', $placed), + "a standby replaying VACUUM's frees logged in a fold's window reads every row at its primary's tuple id" +); +$vacuumed->stop; +$vacuuming->stop; + +$logged->stop; + +done_testing(); diff --git a/src/doltgres/test/t/135_moved_takes.pl b/src/doltgres/test/t/135_moved_takes.pl new file mode 100644 index 0000000000000..a65694eec14ae --- /dev/null +++ b/src/doltgres/test/t/135_moved_takes.pl @@ -0,0 +1,258 @@ +# Copyright 2026 Dolthub, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A free space take whose block's entry a spill moved after its caller read +# it finishes on the entry the spill left, not on the one under the memtable +# it read before (bugs row 342), which lost the spilled takes: a VACUUM whose +# block's running writer spills, which checks the entry under its block's +# range, kept locked through its window, before it logs its record and reads +# again outside the window when the entry moved; and a standby's replayed +# leave of the fresh page whose prepare the full budget refuses, the +# startup's relief spilling the replayed writer's takes of that page. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $spilling = PostgreSQL::Test::Cluster->new('vacuum_spilling'); +$spilling->init; +$spilling->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\nautovacuum = off"); +$spilling->start; +$spilling->safe_psql('postgres', + 'CREATE EXTENSION injection_points; CREATE EXTENSION doltgres_test'); + +# freed makes a table whose block 0 holds a folded delete VACUUM frees. +sub freed +{ + my ($table) = @_; + $spilling->safe_psql( + 'postgres', qq{ +CREATE TABLE $table (k int4, v text) WITH (fillfactor = 50); +INSERT INTO $table VALUES (1, repeat('v', 600)), (2, repeat('v', 600)); +DELETE FROM $table WHERE k = 1; +SELECT doltgres_test_fold('$table'); +}); +} +freed('spilled_block'); +my $spilling_writer = $spilling->background_psql('postgres'); +$spilling_writer->query_safe( + "BEGIN; INSERT INTO spilled_block VALUES (3, repeat('v', 600))"); +my $spilled_vacuum = + $spilling->background_psql('postgres', on_error_stop => 0); +$spilled_vacuum->query_safe('SELECT injection_points_set_local()'); +$spilled_vacuum->query_safe( + "SELECT injection_points_attach('doltgres-vacuum-read', 'wait')"); +$spilled_vacuum->query_until(qr/started/, + "\\echo started\nSELECT doltgres_test_vacuum('spilled_block', 'auto');\n" +); +$spilling->wait_for_event('client backend', 'doltgres-vacuum-read'); +is( join( + ' ', + map { + $spilling_writer->query_safe( + 'SELECT doltgres_test_delta(61, 0, 0)') + } 1 .. 3), + '1 1 0', + "the block's writer spills its rows and then its free space takes while VACUUM waits after its read" +); +$spilling->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-vacuum-read'); SELECT injection_points_wakeup('doltgres-vacuum-read')" +); +$spilled_vacuum->query_safe('SELECT 1'); +$spilled_vacuum->quit; +$spilling_writer->query_safe('COMMIT'); +$spilling_writer->quit; +freed('unspilled_block'); +$spilling->safe_psql( + 'postgres', q{ +INSERT INTO unspilled_block VALUES (3, repeat('v', 600)); +SELECT doltgres_test_vacuum('unspilled_block', 'auto'); +}); +my $spilled_entry = q{SELECT doltgres_test_free_space('%s', 0)}; +is( $spilling->safe_psql( + 'postgres', sprintf($spilled_entry, 'spilled_block')), + $spilling->safe_psql( + 'postgres', sprintf($spilled_entry, 'unspilled_block')), + "VACUUM's give-back after a spill moved its block's takes leaves the entry an unspilled VACUUM leaves" +); + +# The entry's move found before VACUUM's window sends it to read the block +# again outside the window, where an error is an ordinary one that fails the +# VACUUM alone, and a later VACUUM gives the place back as an unspilled one. +freed('erred_block'); +$spilling_writer = $spilling->background_psql('postgres'); +$spilling_writer->query_safe( + "BEGIN; INSERT INTO erred_block VALUES (3, repeat('v', 600))"); +$spilled_vacuum = $spilling->background_psql('postgres', on_error_stop => 0); +$spilled_vacuum->query_safe('SELECT injection_points_set_local()'); +$spilled_vacuum->query_safe( + "SELECT injection_points_attach('doltgres-vacuum-read', 'wait')"); +$spilled_vacuum->query_until(qr/started/, + "\\echo started\nSELECT doltgres_test_vacuum('erred_block', 'auto');\n"); +$spilling->wait_for_event('client backend', 'doltgres-vacuum-read'); +$spilling_writer->query_safe('SELECT doltgres_test_delta(61, 0, 0)') + for 1 .. 3; +$spilling->safe_psql( + 'postgres', q{ +SELECT injection_points_detach('doltgres-vacuum-read'); +SELECT injection_points_attach('doltgres-vacuum-read', 'error'); +SELECT injection_points_wakeup('doltgres-vacuum-read'); +}); +$spilled_vacuum->query('SELECT 1'); +$spilling->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-vacuum-read')"); +like( + $spilled_vacuum->{stderr}, + qr/ERROR: error triggered for injection point doltgres-vacuum-read/, + "an error at VACUUM's read again after its block's entry moved fails the VACUUM as an ordinary error" +); +$spilled_vacuum->quit; +$spilling_writer->query_safe('COMMIT'); +$spilling_writer->quit; +$spilling->safe_psql('postgres', + "SELECT doltgres_test_vacuum('erred_block', 'auto')"); +is( $spilling->safe_psql('postgres', sprintf($spilled_entry, 'erred_block')), + $spilling->safe_psql( + 'postgres', sprintf($spilled_entry, 'unspilled_block')), + "a VACUUM after the failed one leaves the entry an unspilled VACUUM leaves" +); + +# VACUUM keeps its block's range locked from before its window to its last +# take there, so a spill's take-out of the block's free space takes that +# begins while VACUUM is in its window waits on that lock, which VACUUM +# releases waiting on nothing, and both finish. +# toasted makes a table whose block 0 holds a folded delete of a row with a +# TOAST value, whose take-down VACUUM pauses in its window before. +sub toasted +{ + my ($table) = @_; + $spilling->safe_psql( + 'postgres', qq{ +CREATE TABLE $table (k int4, v text); +ALTER TABLE $table ALTER COLUMN v SET STORAGE EXTERNAL; +INSERT INTO $table VALUES (1, repeat('t', 3000)), (2, repeat('t', 3000)); +DELETE FROM $table WHERE k = 1; +SELECT doltgres_test_fold('$table'); +}); +} +toasted('locked_block'); +$spilling_writer = $spilling->background_psql('postgres'); +$spilling_writer->query_safe( + "BEGIN; INSERT INTO locked_block VALUES (3, 'short')"); +$spilling_writer->query_safe('SELECT doltgres_test_delta(61, 0, 0)'); +$spilled_vacuum = $spilling->background_psql('postgres'); +$spilled_vacuum->query_safe( + "SELECT injection_points_set_local(); SELECT injection_points_attach('doltgres-vacuum-logged', 'wait')" +); +$spilled_vacuum->query_until(qr/started/, + "\\echo started\nSELECT doltgres_test_vacuum('locked_block', 'auto');\n"); +$spilling->wait_for_event('client backend', 'doltgres-vacuum-logged'); +my $writer_pid = $spilling_writer->query_safe('SELECT pg_backend_pid()'); +$spilling_writer->query_until(qr/spilling/, + "\\echo spilling\nSELECT doltgres_test_delta(61, 0, 0);\n"); +ok( $spilling->poll_query_until( + 'postgres', + "SELECT wait_event_type = 'LWLock' FROM pg_stat_activity WHERE pid = $writer_pid" + ), + "a spill's take-out of the block's takes waits on the range VACUUM keeps through its window" +); +$spilling->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-vacuum-logged'); SELECT injection_points_wakeup('doltgres-vacuum-logged')" +); +like($spilled_vacuum->query_safe('SELECT 1'), + qr/^1\z/m, "VACUUM finishes its window"); +is($spilling_writer->query_safe('SELECT 1'), + "1\n1", "and then the spill finishes, having spilled"); +$spilled_vacuum->quit; +$spilling_writer->query_safe('COMMIT'); +$spilling_writer->quit; +toasted('unlocked_block'); +$spilling->safe_psql( + 'postgres', q{ +INSERT INTO unlocked_block VALUES (3, 'short'); +SELECT doltgres_test_vacuum('unlocked_block', 'auto'); +}); +is( $spilling->safe_psql('postgres', sprintf($spilled_entry, 'locked_block')), + $spilling->safe_psql( + 'postgres', sprintf($spilled_entry, 'unlocked_block')), + "VACUUM and the spill that waited on its range leave the entry an unspilled VACUUM leaves" +); +$spilling->stop; + + +# A 1MB standby's replay of the row that leaves the fresh page holds its jump +# after it read the page's entry, the budget filled, so the jump's refused +# prepare relieves by spilling the replayed writer's free space takes of the +# page; the jump reads the entry the spill left, and the page's entry is its +# primary's. +my $PAGE = (8 << 32) | (1024 - 64); +my $jumping = PostgreSQL::Test::Cluster->new('jumping'); +$jumping->init(allows_streaming => 1); +$jumping->append_conf('postgresql.conf', + "fold_threshold = 1GB\nfold_timeout = 3600s\nautovacuum = off\ncheckpoint_timeout = 1h" +); +$jumping->start; +$jumping->safe_psql('postgres', + 'CREATE EXTENSION injection_points; CREATE EXTENSION doltgres_test'); +$jumping->backup('jumping_backup'); +my $jumping_standby = PostgreSQL::Test::Cluster->new('jumping_standby'); +$jumping_standby->init_from_backup($jumping, 'jumping_backup', + has_streaming => 1); +$jumping_standby->append_conf('postgresql.conf', + "delta_memory = 1MB\nfold_threshold = 1GB\nfold_timeout = 3600s\ncheckpoint_timeout = 1h" +); +$jumping_standby->start; +$jumping->safe_psql('postgres', 'CREATE TABLE jumped (k int4)'); +$jumping->wait_for_catchup($jumping_standby); +$jumping_standby->safe_psql('postgres', + "SELECT injection_points_attach('doltgres-insert-jump-seen', 'wait')"); +$jumping->safe_psql('postgres', + 'INSERT INTO jumped SELECT g FROM generate_series(1, 300) g'); +my $jumped = + $jumping->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$jumping_standby->wait_for_event('startup', 'doltgres-insert-jump-seen'); +my $full = $jumping_standby->safe_psql('postgres', + 'SELECT doltgres_test_shared(0, 0, 0)'); +$jumping_standby->safe_psql('postgres', + "SELECT doltgres_test_delta(20, $full, $PAGE)"); +$jumping_standby->safe_psql('postgres', + "SELECT injection_points_detach('doltgres-insert-jump-seen'); SELECT injection_points_wakeup('doltgres-insert-jump-seen')" +); +ok( $jumping_standby->poll_query_until( + 'postgres', + "SELECT pg_last_wal_replay_lsn() >= '$jumped'::pg_lsn OR (SELECT count(*) = 1 FROM pg_stat_activity WHERE backend_type = 'startup' AND wait_event = 'versioned delta budget')" + ), + "the standby's startup goes past its refused jump, or waits at its full budget with its relief spent" +); +$jumping_standby->safe_psql('postgres', + "SELECT doltgres_test_delta(21, $full, 0)"); +$jumping->wait_for_catchup($jumping_standby); +my $jumped_entries = + q{SELECT string_agg(b || ':' || doltgres_test_free_space('jumped', b), ' ' ORDER BY b) FROM generate_series(0, (pg_relation_size('jumped') / 8192 - 1)::int4) b}; +is( $jumping_standby->safe_psql('postgres', $jumped_entries), + $jumping->safe_psql('postgres', $jumped_entries), + "a 1MB standby whose replayed leave's read a spill at its refused prepare moved holds its primary's free space entries" +); +$jumping_standby->stop; +$jumping->stop; + +done_testing(); diff --git a/src/doltgres/trees/c/cursor.c b/src/doltgres/trees/c/cursor.c index 24aa134797077..de4783430fc6b 100644 --- a/src/doltgres/trees/c/cursor.c +++ b/src/doltgres/trees/c/cursor.c @@ -1070,6 +1070,34 @@ trees_cursor_Cursor_index(const TreesCursorCursor *self) return trees_cursor_Cursor_bottom(self)->idx; } +/* + * trees_cursor_Cursor_find_in_node moves the cursor to the entry of its own + * node whose key is the key, by a search of that node alone, and answers + * whether the node holds one, leaving the cursor where it was otherwise: a + * tree's keys are unique, so an entry found is the tree's entry of the key. + */ +bool +trees_cursor_Cursor_find_in_node(TreesCursorCursor *self, HostIoIo *io, const uint8 *key, size_t key_len, + const TreesKeyOrder *order, bool *result, ChunksErrorError *error) +{ + TreesNodeNode node = trees_cursor_Cursor_node(self); + size_t at; + StdBytes found; + + *result = false; + if (!trees_node_Node_search(&node, io, key, key_len, order, &at, error)) + return false; + if (at >= trees_node_Node_len(&node)) + return true; + found = trees_node_Node_key(&node, at); + if (found.len != key_len || memcmp(found.ptr, key, key_len) != 0) + return true; + STD_VEC_AT(&self->path, TreesCursorFrame, std_wrapping_sub_usize(self->path.len, 1)).idx = at; + self->valid = true; + *result = true; + return true; +} + /* trees_cursor_Cursor_key returns the key of the entry the cursor is at. */ StdBytes trees_cursor_Cursor_key(const TreesCursorCursor *self) @@ -1690,7 +1718,7 @@ trees_cursor_LeafStamps_Default_default(void) return result; } -/* trees_cursor_LeafStamps_of_frame_0_t is of's state across its pad: the pair it took from the cache, while live. */ +/* trees_cursor_LeafStamps_of_frame_0_t is of's and stamp_of's state across their pad: the pair taken from the cache, while live, and their answers. */ typedef struct trees_cursor_LeafStamps_of_frame_0_t { TreesCursorLeafStamps *self; @@ -1700,22 +1728,23 @@ typedef struct trees_cursor_LeafStamps_of_frame_0_t trees_TupleTreeArcStamps_t taken; bool taken_live; StdVec *result; + uint64 tuple; + uint64 *copies; + bool *some; + ChunksTrailerStamp *stamp; ChunksErrorError *error; bool ok; } trees_cursor_LeafStamps_of_frame_0_t; /* - * trees_cursor_LeafStamps_of_body_0 is of's body: the cached pair taken, put - * back when it is the cursor's leaf's, or else the leaf's stamps read and - * stored, the taken pair dropped after, at the end of the let. + * trees_cursor_LeafStamps_read is of's and stamp_of's read: the cached pair + * taken, put back when it is the cursor's leaf's, or else the leaf's stamps + * read and stored, the taken pair dropped after, at the end of the let. */ -static void -trees_cursor_LeafStamps_of_body_0(void *frame_) +static bool +trees_cursor_LeafStamps_read(trees_cursor_LeafStamps_of_frame_0_t *frame) { - trees_cursor_LeafStamps_of_frame_0_t *frame = frame_; TreesStoreTree node = trees_cursor_Cursor_node_tree(frame->cursor); - const ChunksTrailerStamps *stamps; - size_t index; frame->taken_live = frame->self->has_f0; frame->taken = frame->self->f0; @@ -1737,8 +1766,7 @@ trees_cursor_LeafStamps_of_body_0(void *frame_) frame->taken_live = false; trees_cursor_ArcStamps_drop(frame->taken.f1); } - frame->ok = false; - return; + return false; } frame->self->f0.f0 = node; frame->self->f0.f1 = read; @@ -1749,9 +1777,40 @@ trees_cursor_LeafStamps_of_body_0(void *frame_) trees_cursor_ArcStamps_drop(frame->taken.f1); } } + return true; +} + +/* trees_cursor_LeafStamps_of_body_0 is of's body. */ +static void +trees_cursor_LeafStamps_of_body_0(void *frame_) +{ + trees_cursor_LeafStamps_of_frame_0_t *frame = frame_; + + if (!trees_cursor_LeafStamps_read(frame)) + return; + chunks_trailer_Stamps_entry(std_arc_Deref_deref(frame->self->f0.f1, _Alignof(ChunksTrailerStamps)), + trees_cursor_Cursor_index(frame->cursor), frame->result); + frame->ok = true; +} + +/* trees_cursor_LeafStamps_stamp_of_body_0 is stamp_of's body. */ +static void +trees_cursor_LeafStamps_stamp_of_body_0(void *frame_) +{ + trees_cursor_LeafStamps_of_frame_0_t *frame = frame_; + const ChunksTrailerStamps *stamps; + size_t index; + const uint32 *values; + size_t values_len; + + if (!trees_cursor_LeafStamps_read(frame)) + return; stamps = std_arc_Deref_deref(frame->self->f0.f1, _Alignof(ChunksTrailerStamps)); index = trees_cursor_Cursor_index(frame->cursor); - chunks_trailer_Stamps_entry(stamps, index, frame->result); + *frame->copies = chunks_trailer_Stamps_entry_len(stamps, index); + *frame->some = chunks_trailer_Stamps_stamp_of(stamps, index, frame->tuple, frame->stamp); + values = chunks_trailer_Stamps_values(stamps, index, &values_len); + std_slice_to_vec_Checked(frame->result, values, values_len, sizeof(uint32), _Alignof(uint32)); frame->ok = true; } @@ -1781,6 +1840,25 @@ trees_cursor_LeafStamps_of(TreesCursorLeafStamps *self, HostIoIo *io, TreesPinPi return frame.ok; } +/* + * trees_cursor_LeafStamps_stamp_of returns the number of row stamps of the + * leaf entry a cursor is at, one for each copy of a row it counts, its stamp + * of a tuple id, or nothing when it has none, by a search of the leaf's + * stamps, and its row's value ids as Cursor's values gives them, the leaf's + * stamps read only when the cursor is in another leaf than the last. + */ +bool +trees_cursor_LeafStamps_stamp_of(TreesCursorLeafStamps *self, HostIoIo *io, TreesPinPin *pin, + const TreesCursorCursor *cursor, uint64 tuple, uint64 *copies, bool *some, + ChunksTrailerStamp *result, StdVec *values, ChunksErrorError *error) +{ + trees_cursor_LeafStamps_of_frame_0_t frame = {.self = self, .io = io, .pin = pin, .cursor = cursor, + .result = values, .tuple = tuple, .copies = copies, .some = some, .stamp = result, .error = error}; + + std_landing(&frame, trees_cursor_LeafStamps_stamp_of_body_0, trees_cursor_LeafStamps_of_pad_0); + return frame.ok; +} + /* trees_cursor_LeafStamps_drop is LeafStamps' drop glue: its leaf's stamps, when it holds them. */ void trees_cursor_LeafStamps_drop(TreesCursorLeafStamps *self) diff --git a/src/doltgres/txn/c/base.c b/src/doltgres/txn/c/base.c index 940ef83a33151..62c1c53d9b8aa 100644 --- a/src/doltgres/txn/c/base.c +++ b/src/doltgres/txn/c/base.c @@ -45,9 +45,9 @@ #define RUST_FILE "txn" STD_SEP "src" STD_SEP "base.rs" /* - * The record's field numbers, in the format's order, TABLE, MAX_ENDED, and - * CHAIN always present, CHAIN the highest-numbered and last, so a record cut - * short anywhere is refused. + * The record's field numbers, in the format's order, TABLE, MAX_ENDED, CHAIN, + * and READ always present, READ the highest-numbered and last, so a record + * cut short anywhere is refused. */ #define TXN_BASE_ROOT 1 #define TXN_BASE_FOLDED 2 @@ -64,6 +64,7 @@ #define TXN_BASE_PENDING 13 #define TXN_BASE_MAX_ENDED 14 #define TXN_BASE_CHAIN 16 +#define TXN_BASE_READ 18 /* TXN_BASE_ATTNUM_MAX is Postgres' MaxHeapAttributeNumber, the largest attribute number a key's column has. */ #define TXN_BASE_ATTNUM_MAX UINT64CONST(1600) @@ -316,7 +317,7 @@ txn_base_Base_PartialEq_eq(const TxnBaseBase *self, const TxnBaseBase *other) txn_base_Key_PartialEq_eq(&self->key, &other->key) && txn_base_tree_eq(self->has_pending, &self->pending, other->has_pending, &other->pending) && self->max_ended == other->max_ended && - txn_base_tree_eq(self->has_chain, &self->chain, other->has_chain, &other->chain); + txn_base_tree_eq(self->has_chain, &self->chain, other->has_chain, &other->chain) && self->read == other->read; } /* txn_base_Base_Debug_fmt is Base's derived Debug. */ @@ -347,6 +348,7 @@ txn_base_Base_Debug_fmt(const void *value, StdFormatter *f) std_DebugStruct_field(&s, "pending", 7, &pending, std_option_Debug_fmt); std_DebugStruct_field(&s, "max_ended", 9, &self->max_ended, std_u64_Debug_fmt); std_DebugStruct_field(&s, "chain", 5, &chain, std_option_Debug_fmt); + std_DebugStruct_field(&s, "read", 4, &self->read, std_u64_Debug_fmt); return std_DebugStruct_finish(&s); } @@ -905,7 +907,9 @@ txn_base_pair_value(const uint8 *key, size_t key_len, uint32 heap_bytes, StdVec /* * txn_base_pending_value returns a pending tuple id's entry's value in the * base's pending tree: its row's ender's full xid, 0 when the ender aborted, - * then its heap bytes, then the value ids its row names, each little-endian. + * with TXN_BASE_OWN set when the ender is the row's own writer, then its heap + * bytes, 0 when a prune already gave them back, then the value ids its row + * names, each little-endian. */ void txn_base_pending_value(uint64 ender, uint32 heap_bytes, const uint32 *ids, size_t ids_len, StdVec *result) @@ -924,7 +928,7 @@ txn_base_pending_value(uint64 ender, uint32 heap_bytes, const uint32 *ids, size_ /* * txn_base_pending_read reads a pending tuple id's entry's value as * pending_value writes it, refusing a length that isn't 12 bytes and a whole - * number of value ids, and heap bytes of 0, as data corruption. + * number of value ids as data corruption. */ bool txn_base_pending_read(const uint8 *value, size_t value_len, uint64 *result_0, uint32 *result_1, StdVec *result_2, @@ -932,7 +936,7 @@ txn_base_pending_read(const uint8 *value, size_t value_len, uint64 *result_0, ui { size_t ids; - if (value_len < 12 || (value_len - 12) % 4 != 0 || std_u32_from_le_bytes(value + 8) == 0) + if (value_len < 12 || (value_len - 12) % 4 != 0) { *error = TXN_BASE_FORK("XX001", "a pending tuple id's entry is corrupt"); return false; @@ -1148,6 +1152,7 @@ txn_base_encode_body(void *frame_) txn_base_address(&frame->writer, TXN_BASE_CHAIN, true, &self->chain); else chunks_record_RecordWriter_field(&frame->writer, TXN_BASE_CHAIN, TXN_BASE_NO_BYTES, 0); + chunks_record_RecordWriter_uint(&frame->writer, TXN_BASE_READ, self->read); frame->writer_live = false; chunks_record_RecordWriter_finish_value(&frame->writer, frame->result_0, frame->result_1); } @@ -1230,13 +1235,13 @@ txn_base_decode_fields_free(txn_base_decode_frame_t *frame) * outside a range and isn't another number. */ static bool -txn_base_decode_outside(const txn_base_decode_frame_t *frame, uint32 least, uint32 most, uint32 other) +txn_base_decode_outside(const txn_base_decode_frame_t *frame, uint32 least, uint32 most, uint32 other, uint32 last) { for (size_t i = 0; i < frame->fields.len; i++) { uint32 number = STD_VEC_AT(&frame->fields, ChunksRecordField, i).number; - if ((number < least || number > most) && number != other) + if ((number < least || number > most) && number != other && number != last) return true; } return false; @@ -1416,7 +1421,7 @@ txn_base_decode_body(void *frame_) frame->record_live = true; if (!txn_base_decode_fields(frame, frame->bytes, frame->bytes_len)) return; - if (txn_base_decode_outside(frame, TXN_BASE_ROOT, TXN_BASE_MAX_ENDED, TXN_BASE_CHAIN)) + if (txn_base_decode_outside(frame, TXN_BASE_ROOT, TXN_BASE_MAX_ENDED, TXN_BASE_CHAIN, TXN_BASE_READ)) { *frame->error = txn_base_corrupt(); return; @@ -1431,7 +1436,7 @@ txn_base_decode_body(void *frame_) if (!txn_base_decode_nested(frame, &frame->record, sizes, &frame->held, &frame->held_live) || !txn_base_decode_fields(frame, sizes->bytes, sizes->bytes_len)) return; - if (txn_base_decode_outside(frame, 1, 5, 1)) + if (txn_base_decode_outside(frame, 1, 5, 1, 1)) { *frame->error = txn_base_corrupt(); return; @@ -1483,6 +1488,8 @@ txn_base_decode_body(void *frame_) } if (chain->bytes_len != 0 && !txn_base_decode_tree(frame, TXN_BASE_CHAIN, &base.has_chain, &base.chain)) return; + if (!txn_base_record_uint(&frame->record, TXN_BASE_READ, 0, base.folded, &base.read, frame->error)) + return; frame->columns_live = false; *frame->result = base; frame->ok = true; @@ -1572,6 +1579,7 @@ txn_base_tests_full(void) .has_pending = true, .pending = txn_base_tests_tree(4), .max_ended = PG_UINT64_MAX, .has_chain = true, .chain = txn_base_tests_tree(5), + .read = PG_UINT64_MAX, }; return base; @@ -1672,8 +1680,8 @@ txn_base_tests_round_trips(uint32 line, const TxnBaseBase *base) void test_txn_base_tests_a_base_record_round_trips_at_its_limits(void) { - TxnBaseBase bases[4] = {txn_base_tests_full(), txn_base_tests_empty(), txn_base_tests_full(), - txn_base_tests_empty()}; + TxnBaseBase bases[5] = {txn_base_tests_full(), txn_base_tests_empty(), txn_base_tests_full(), + txn_base_tests_empty(), txn_base_tests_empty()}; bases[2].has_index = false; bases[2].index = 0; @@ -1681,7 +1689,9 @@ test_txn_base_tests_a_base_record_round_trips_at_its_limits(void) bases[3].index = 0; bases[3].has_root = true; bases[3].root = txn_base_tests_tree(9); - for (int i = 0; i < 4; i++) + bases[4].folded = 200; + bases[4].read = 149; + for (int i = 0; i < lengthof(bases); i++) txn_base_tests_round_trips(605, &bases[i]); } @@ -1765,6 +1775,7 @@ txn_base_tests_write_plain(ChunksRecordRecordWriter *writer, uint32 number) case TXN_BASE_INDEX: case TXN_BASE_KEY: case 15: + case 17: break; case TXN_BASE_CHAIN: chunks_record_RecordWriter_field(writer, TXN_BASE_CHAIN, TXN_BASE_NO_BYTES, 0); @@ -1800,13 +1811,13 @@ txn_base_tests_write_changed(ChunksRecordRecordWriter *writer, const TxnBaseTest txn_base_tests_write_plain(writer, number); } -/* txn_base_tests_write_form writes every field from ROOT to CHAIN in a form, but a skipped one, as a value. */ +/* txn_base_tests_write_form writes every field from ROOT to READ in a form, but a skipped one, as a value. */ static void txn_base_tests_write_form(uint32 skip, const TxnBaseTestsForm *form, StdVec *bytes, StdVec *addresses) { ChunksRecordRecordWriter writer = chunks_record_RecordWriter_new(); - for (uint32 number = TXN_BASE_ROOT; number <= TXN_BASE_CHAIN; number++) + for (uint32 number = TXN_BASE_ROOT; number <= TXN_BASE_READ; number++) if (number != skip) txn_base_tests_write_changed(&writer, form, number); chunks_record_RecordWriter_finish_value(&writer, bytes, addresses); @@ -1822,9 +1833,9 @@ txn_base_tests_write_form(uint32 skip, const TxnBaseTestsForm *form, StdVec *byt void test_txn_base_tests_every_other_form_of_a_base_record_is_corrupt(void) { - static const uint32 required[] = {TXN_BASE_FOLDED, TXN_BASE_MAX_FOLDED, TXN_BASE_EPOCH, TXN_BASE_COUNTER, TXN_BASE_FRESH_END, TXN_BASE_SIZES, TXN_BASE_TABLE, TXN_BASE_MAX_ENDED, TXN_BASE_CHAIN}; + static const uint32 required[] = {TXN_BASE_FOLDED, TXN_BASE_MAX_FOLDED, TXN_BASE_EPOCH, TXN_BASE_COUNTER, TXN_BASE_FRESH_END, TXN_BASE_SIZES, TXN_BASE_TABLE, TXN_BASE_MAX_ENDED, TXN_BASE_CHAIN, TXN_BASE_READ}; static const TxnBaseTestsForm past[] = {{TXN_BASE_TESTS_FORM_PAST, TXN_BASE_EPOCH, TXN_BASE_U32_MAX}, {TXN_BASE_TESTS_FORM_PAST, TXN_BASE_COUNTER, TXN_BASE_TUPLE_ID_MAX}, - {TXN_BASE_TESTS_FORM_PAST, TXN_BASE_FRESH_END, TXN_BASE_TUPLE_ID_MAX}, {TXN_BASE_TESTS_FORM_PAST, TXN_BASE_TABLE, TXN_BASE_U32_MAX}, {TXN_BASE_TESTS_FORM_PAST, TXN_BASE_INDEX, TXN_BASE_U32_MAX}}; + {TXN_BASE_TESTS_FORM_PAST, TXN_BASE_FRESH_END, TXN_BASE_TUPLE_ID_MAX}, {TXN_BASE_TESTS_FORM_PAST, TXN_BASE_TABLE, TXN_BASE_U32_MAX}, {TXN_BASE_TESTS_FORM_PAST, TXN_BASE_INDEX, TXN_BASE_U32_MAX}, {TXN_BASE_TESTS_FORM_PAST, TXN_BASE_READ, 0}}; static const TxnBaseTestsForm plain = {TXN_BASE_TESTS_FORM_PLAIN}; StdVec bytes; StdVec addresses; @@ -1925,6 +1936,7 @@ txn_base_tests_keyed(const TxnBaseTestsKeyedCase *keyed_case, StdVec *bytes, Std chunks_record_RecordWriter_uint(&writer, TXN_BASE_TABLE, 0); chunks_record_RecordWriter_uint(&writer, TXN_BASE_MAX_ENDED, 0); chunks_record_RecordWriter_field(&writer, TXN_BASE_CHAIN, TXN_BASE_NO_BYTES, 0); + chunks_record_RecordWriter_uint(&writer, TXN_BASE_READ, 0); chunks_record_RecordWriter_finish_value(&writer, bytes, addresses); } diff --git a/src/doltgres/txn/c/blobs.c b/src/doltgres/txn/c/blobs.c index 6083729a13310..b25e7048ce765 100644 --- a/src/doltgres/txn/c/blobs.c +++ b/src/doltgres/txn/c/blobs.c @@ -41,6 +41,18 @@ #include "doltgres/txn/spill.h" #include "doltgres/txn/written.h" +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/repo/lib.h" +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" +#include "doltgres/trees/blob.h" +#include "doltgres/trees/chunker.h" +#include "doltgres/txn/status.h" + +/* RUST_FILE is the file the tests' asserts name, blobs.rs. */ +#define RUST_FILE "txn" STD_SEP "src" STD_SEP "blobs.rs" +#endif + /* txn_blobs_unread returns the error of a read of a cut, which a blob's write never makes. */ static ChunksErrorError txn_blobs_unread(const ChunksAddressAddress *address) @@ -768,3 +780,506 @@ txn_blobs_redo(HostIoIo *io, const HostSharedHeld *deltas, TxnRecordChunkWrite c std_landing(&frame, txn_blobs_redo_body_0, txn_blobs_redo_pad_0); return frame.ok; } + +#ifdef DOLTGRES_TEST_TXN +/* TXN_BLOBS_TESTS_UNWRAP is Result::unwrap's panic text. */ +#define TXN_BLOBS_TESTS_UNWRAP "called `Result::unwrap()` on an `Err` value" + +/* txn_blobs_tests_ok unwraps a host result at a line and column. */ +static void +txn_blobs_tests_ok(bool ok, HostError *error, uint32 line, uint32 col) +{ + if (!ok) + STD_RESULT_EXPECT_FAILED(line, col, TXN_BLOBS_TESTS_UNWRAP, error, host_Error_Debug_fmt); +} + +/* txn_blobs_tests_unpacked is the tests' packed closure: the packs hold no chunk. */ +static bool +txn_blobs_tests_unpacked(void *captures, HostIoIo *io, const ChunksAddressAddress *address, bool *result, + HostError *error) +{ + (void) captures; + (void) io; + (void) address; + (void) error; + *result = false; + return true; +} + +/* txn_blobs_tests_current returns the running actor's transaction as a chunk's writer, its xid assigned. */ +static TxnMemtableVersionWriter +txn_blobs_tests_current(HostIoIo *io) +{ + HostXid sub; + HostXid top; + HostError error; + TxnMemtableVersionWriter writer = {0}; + + txn_blobs_tests_ok(host_current_writer(HOST_IO_IO_IO_DOORS(io), &sub, &top, &error), &error, 289, 51); + writer.xid = sub.f0; + writer.has_top = sub.f0 != top.f0; + writer.top = top.f0; + return writer; +} + +/* txn_blobs_tests_chunk returns a test's one chunk of some bytes, by its address. */ +static StdVec +txn_blobs_tests_chunk(const uint8 *bytes, size_t bytes_len) +{ + StdVec chunks; + txn_TupleAddressVecU8_t chunk = {.f0 = chunks_address_Address_of(bytes, bytes_len)}; + + std_vec_new(&chunks, sizeof(txn_TupleAddressVecU8_t), _Alignof(txn_TupleAddressVecU8_t)); + std_slice_to_vec_Checked(&chunk.f1, bytes, bytes_len, 1, 1); + std_vec_push_Checked(&chunks, &chunk, sizeof(chunk), _Alignof(txn_TupleAddressVecU8_t)); + return chunks; +} + +/* txn_blobs_tests_chunked returns a test's chunks, each of 100 bytes of one value, by their addresses. */ +static StdVec +txn_blobs_tests_chunked(uint8 first, uint8 end) +{ + StdVec chunks; + + std_vec_new(&chunks, sizeof(txn_TupleAddressVecU8_t), _Alignof(txn_TupleAddressVecU8_t)); + for (uint32 value = first; value < end; value++) + { + uint8 bytes[100]; + StdVec one; + + memset(bytes, (int) value, sizeof(bytes)); + one = txn_blobs_tests_chunk(bytes, sizeof(bytes)); + std_vec_push_Checked(&chunks, one.ptr, sizeof(txn_TupleAddressVecU8_t), _Alignof(txn_TupleAddressVecU8_t)); + std_vec_free_Checked(&one, sizeof(txn_TupleAddressVecU8_t), _Alignof(txn_TupleAddressVecU8_t)); + } + return chunks; +} + +/* txn_blobs_tests_chunks_drop drops a test's chunks. */ +static void +txn_blobs_tests_chunks_drop(StdVec *chunks) +{ + for (size_t k = 0; k < chunks->len; k++) + std_vec_free_Checked(&STD_VEC_AT(chunks, txn_TupleAddressVecU8_t, k).f1, 1, 1); + std_vec_free_Checked(chunks, sizeof(txn_TupleAddressVecU8_t), _Alignof(txn_TupleAddressVecU8_t)); +} + +/* txn_blobs_tests_deltas returns a repository's working state's deltas, which unwraps at a line and column. */ +static HostSharedHeld +txn_blobs_tests_deltas(HostIoIo *io, uint32 repository, uint32 line, uint32 col) +{ + HostSharedHeld held; + HostError error; + + txn_blobs_tests_ok(txn_memtable_branch_deltas(io, repository, HOST_PG_DEFAULT, &held, &error), &error, line, col); + return held; +} + +/* txn_blobs_tests_written writes chunks under a writer, taking them, which unwraps at a line and column. */ +static void +txn_blobs_tests_written(HostIoIo *io, const HostSharedHeld *deltas, uint32 repository, TxnMemtableVersionWriter writer, + StdVec chunks, bool logged, uint32 line, uint32 col) +{ + HostError error; + + txn_blobs_tests_ok(txn_blobs_written(io, deltas, repository, writer, &chunks, logged, txn_blobs_tests_unpacked, + NULL, NULL, &error), &error, line, col); +} + +/* txn_blobs_tests_got reads a chunk from the delta, answering whether it holds one, into bytes when given. */ +static bool +txn_blobs_tests_got(HostIoIo *io, const HostSharedHeld *deltas, const ChunksAddressAddress *address, StdVec *bytes, + uint32 line, uint32 col) +{ + StdVec read; + bool some; + HostError error; + + txn_blobs_tests_ok(txn_memtable_branch_get(io, host_shared_Held_record(deltas), address->f0, sizeof(address->f0), + &some, &read, &error), &error, line, col); + if (some && bytes != NULL) + *bytes = read; + else if (some) + std_vec_free_Checked(&read, 1, 1); + return some; +} + +/* txn_blobs_tests_durable reports whether the delta's chunk is durable, which unwraps at a line and column. */ +static bool +txn_blobs_tests_durable(HostIoIo *io, const HostSharedHeld *deltas, const ChunksAddressAddress *address, uint32 line, + uint32 col) +{ + bool durable; + HostError error; + + txn_blobs_tests_ok(txn_memtable_branch_durable(io, host_shared_Held_record(deltas), address->f0, + sizeof(address->f0), &durable, &error), &error, line, col); + return durable; +} + +/* txn_blobs_tests_spilled reads a chunk from its transaction's chunk spill, answering whether it holds one, into bytes when given. */ +static bool +txn_blobs_tests_spilled(HostIoIo *io, const HostSharedHeld *deltas, const ChunksAddressAddress *address, StdVec *bytes, + uint32 line, uint32 col) +{ + StdVec read; + bool some; + HostError error; + + txn_blobs_tests_ok(txn_spill_chunk(io, host_shared_Held_record(deltas), address->f0, sizeof(address->f0), &some, + &read, &error), &error, line, col); + if (some && bytes != NULL) + *bytes = read; + else if (some) + std_vec_free_Checked(&read, 1, 1); + return some; +} + +/* txn_blobs_tests_logged_t is a logged chunk record of a repository with its end. */ +typedef struct txn_blobs_tests_logged_t +{ + TxnRecordChunkWrite chunk; + uint64 end; +} txn_blobs_tests_logged_t; + +/* txn_blobs_tests_logged returns the chunk records the simulation logged for a repository, with their ends. */ +static StdVec +txn_blobs_tests_logged(uint32 repository) +{ + StdVec logged = simc_logged(); + StdVec records; + + std_vec_new(&records, sizeof(txn_blobs_tests_logged_t), _Alignof(txn_blobs_tests_logged_t)); + for (size_t k = 0; k < logged.len; k++) + { + simc_TupleU8VecU8Lsn_t *one = &STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, k); + txn_blobs_tests_logged_t record = {.end = one->f2}; + HostWalKind kind; + HostError error; + + if (host_WalKind_of(one->f0, &kind) && kind == HOST_WAL_KIND_CHUNK) + { + if (!txn_record_ChunkWrite_decode(one->f1.ptr, one->f1.len, &record.chunk, &error)) + host_Error_drop(&error); + else if (record.chunk.repository != repository) + txn_record_ChunkWrite_drop(&record.chunk); + else + std_vec_push_Checked(&records, &record, sizeof(record), _Alignof(txn_blobs_tests_logged_t)); + } + std_vec_free_Checked(&one->f1, 1, 1); + } + std_vec_free_Checked(&logged, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + return records; +} + +/* txn_blobs_tests_logged_drop frees what logged returned. */ +static void +txn_blobs_tests_logged_drop(StdVec *records) +{ + for (size_t k = 0; k < records->len; k++) + txn_record_ChunkWrite_drop(&STD_VEC_AT(records, txn_blobs_tests_logged_t, k).chunk); + std_vec_free_Checked(records, sizeof(txn_blobs_tests_logged_t), _Alignof(txn_blobs_tests_logged_t)); +} + +/* txn_blobs_tests_ended ends the running actor's transaction, committed or aborted, and its written list with it. */ +static void +txn_blobs_tests_ended(HostIoIo *io, bool committed, uint32 line, uint32 col) +{ + HostError error; + + simc_end(committed); + txn_blobs_tests_ok(committed ? txn_written_ended(io, &error) : txn_written_aborted(io, 0, &error), &error, line, col); +} + +/* + * test_txn_blobs_tests_a_logged_write_logs_a_chunk_an_unlogged_one_left_and_redo_makes_it_durable: + * the out-of-line values text's V2, a chunk an unlogged write put into the + * working state's delta is not durable, so a logged write of the same chunk + * pushes and logs its own version, which makes it durable; a second logged + * write passes over it; and redo of the logged records on another working + * state makes the chunk durable there. + */ +void +test_txn_blobs_tests_a_logged_write_logs_a_chunk_an_unlogged_one_left_and_redo_makes_it_durable(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld deltas = txn_blobs_tests_deltas(io, 9320, 228, 73); + HostSharedHeld replica; + ChunksAddressAddress address = chunks_address_Address_of((const uint8 *) "blob", 4); + TxnMemtableVersionWriter writer = txn_blobs_tests_current(io); + StdVec records; + StdVec bytes; + size_t count; + size_t one = 1; + + txn_blobs_tests_written(io, &deltas, 9320, writer, txn_blobs_tests_chunk((const uint8 *) "blob", 4), false, 233, + 83); + STD_ASSERT(234, 9, txn_blobs_tests_got(io, &deltas, &address, NULL, 234, 62), + "branch::get(io, deltas.record(), &address.0).unwrap().is_some()"); + STD_ASSERT(235, 9, !txn_blobs_tests_durable(io, &deltas, &address, 235, 67), + "!branch::durable(io, deltas.record(), &address.0).unwrap()"); + records = txn_blobs_tests_logged(9320); + STD_ASSERT(244, 9, records.len == 0, "logged().is_empty()"); + txn_blobs_tests_logged_drop(&records); + txn_blobs_tests_written(io, &deltas, 9320, writer, txn_blobs_tests_chunk((const uint8 *) "blob", 4), true, 245, + 82); + STD_ASSERT(246, 9, txn_blobs_tests_durable(io, &deltas, &address, 246, 66), + "branch::durable(io, deltas.record(), &address.0).unwrap()"); + txn_blobs_tests_written(io, &deltas, 9320, writer, txn_blobs_tests_chunk((const uint8 *) "blob", 4), true, 247, + 82); + records = txn_blobs_tests_logged(9320); + count = records.len; + STD_ASSERT_EQ(249, 9, count == 1, &count, std_usize_Debug_fmt, &one, std_usize_Debug_fmt); + replica = txn_blobs_tests_deltas(io, 9321, 250, 94); + for (size_t k = 0; k < records.len; k++) + { + txn_blobs_tests_logged_t *record = &STD_VEC_AT(&records, txn_blobs_tests_logged_t, k); + HostError error; + + txn_blobs_tests_ok(txn_blobs_redo(io, &replica, record->chunk, record->end, &error), &error, 252, 44); + } + std_vec_free_Checked(&records, sizeof(txn_blobs_tests_logged_t), _Alignof(txn_blobs_tests_logged_t)); + STD_ASSERT(254, 9, txn_blobs_tests_durable(io, &replica, &address, 254, 67), + "branch::durable(io, replica.record(), &address.0).unwrap()"); + STD_ASSERT(255, 9, txn_blobs_tests_got(io, &replica, &address, &bytes, 255, 66) && bytes.len == 4 && + memcmp(bytes.ptr, "blob", 4) == 0, + "branch::get(io, replica.record(), &address.0).unwrap().as_deref() == Some(&b\"blob\"[..])"); + std_vec_free_Checked(&bytes, 1, 1); + txn_blobs_tests_ended(io, true, 257, 35); + host_shared_Held_drop(&replica); + host_shared_Held_drop(&deltas); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * test_txn_blobs_tests_a_blob_in_the_delta_reads_back_through_blobs: PLAN + * item 5's TOAST read, a blob tree whose chunks a write put into the working + * state's delta reads back whole and in slices through Blobs, and with no + * delta published its chunks aren't found. + */ +void +test_txn_blobs_tests_a_blob_in_the_delta_reads_back_through_blobs(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + StdVec owned; + uint8 *data; + TxnBlobsCut cut; + TreesStoreTreeRef root; + TreesStoreMemoryStore packs = trees_store_MemoryStore_new(); + TxnBlobsBlobs unread; + TxnBlobsBlobs blobs; + HostSharedHeld deltas; + TxnMemtableVersionWriter writer; + StdVec read; + uint8 slice[3000]; + ChunksErrorError failed; + HostError error; + + std_vec_from_elem_zeroed(&owned, 100000, 1, 1); + data = owned.ptr; + for (uint32 i = 0; i < 100000; i++) + data[i] = (uint8) ((i * UINT64CONST(2654435761) & PG_UINT32_MAX) >> 24); + txn_blobs_Cut_Default_default(&cut); + if (!trees_blob_write_blob(io, TXN_BLOBS_CUT_NODE_STORE(&cut), data, 100000, TREES_CHUNKER_DEFAULT_TARGET, &root, + &failed)) + STD_RESULT_EXPECT_FAILED(270, 82, TXN_BLOBS_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + STD_ASSERT_MSG(271, 9, cut.chunks.len > 1, "the blob fit one chunk"); + txn_blobs_tests_ok(txn_blobs_Blobs_new(io, 9322, TREES_STORE_MEMORY_STORE_NODE_STORE(&packs), &unread, &error), + &error, 273, 57); + if (trees_blob_read_blob(io, TXN_BLOBS_BLOBS_NODE_STORE(&unread), TREES_STORE_TREE_REF_NAMED(&root), &read, + &failed)) + STD_ASSERT_MSG(274, 9, false, "a blob no delta holds was read"); + chunks_error_Error_drop(&failed); + txn_blobs_Blobs_drop(&unread); + deltas = txn_blobs_tests_deltas(io, 9322, 275, 73); + writer = txn_blobs_tests_current(io); + txn_blobs_tests_written(io, &deltas, 9322, writer, cut.chunks, false, 277, 89); + txn_blobs_tests_ok(txn_blobs_Blobs_new(io, 9322, TREES_STORE_MEMORY_STORE_NODE_STORE(&packs), &blobs, &error), + &error, 278, 56); + if (!trees_blob_read_blob(io, TXN_BLOBS_BLOBS_NODE_STORE(&blobs), TREES_STORE_TREE_REF_NAMED(&root), &read, + &failed)) + STD_RESULT_EXPECT_FAILED(279, 55, TXN_BLOBS_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + STD_ASSERT(279, 9, read.len == 100000 && memcmp(read.ptr, data, 100000) == 0, + "trees::read_blob(io, &blobs, root).unwrap() == data"); + std_vec_free_Checked(&read, 1, 1); + if (!trees_blob_read_slice(io, TXN_BLOBS_BLOBS_NODE_STORE(&blobs), TREES_STORE_TREE_REF_NAMED(&root), 40000, + slice, sizeof(slice), &failed)) + STD_RESULT_EXPECT_FAILED(281, 65, TXN_BLOBS_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + STD_ASSERT(282, 9, memcmp(slice, data + 40000, sizeof(slice)) == 0, "slice == data[40_000..43_000]"); + txn_blobs_tests_ended(io, true, 284, 35); + txn_blobs_Blobs_drop(&blobs); + host_shared_Held_drop(&deltas); + trees_store_MemoryStore_drop(&packs); + std_vec_free_Checked(&owned, 1, 1); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * test_txn_blobs_tests_a_transactions_chunks_spill_read_back_and_leave_at_its_abort: + * PLAN's chunk spill row, a transaction's chunks spill out of the working + * state's delta into its chunk spill, where spill::chunk reads them back, and + * its abort takes the spill out of the manifest, so none reads back after. + */ +void +test_txn_blobs_tests_a_transactions_chunks_spill_read_back_and_leave_at_its_abort(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedHeld deltas; + TxnMemtableVersionWriter writer; + HostStorage storage = txn_spill_chunks_storage(9324); + StdVec chunks; + bool spilled; + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_create(io, 9324, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(306, 74, TXN_BLOBS_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + deltas = txn_blobs_tests_deltas(io, 9324, 307, 73); + writer = txn_blobs_tests_current(io); + txn_blobs_tests_written(io, &deltas, 9324, writer, txn_blobs_tests_chunked(0, 3), true, 309, 91); + txn_blobs_tests_ok(txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_CHUNKS, writer.xid, &spilled, &error), &error, + 310, 96); + STD_ASSERT(310, 9, spilled, + "spill::spilled(io, &spill::chunks_storage(repository), Of::Chunks, writer.xid).unwrap()"); + chunks = txn_blobs_tests_chunked(0, 3); + for (size_t k = 0; k < chunks.len; k++) + { + txn_TupleAddressVecU8_t *chunk = &STD_VEC_AT(&chunks, txn_TupleAddressVecU8_t, k); + StdVec bytes; + + STD_ASSERT_MSG(312, 13, !txn_blobs_tests_got(io, &deltas, &chunk->f0, NULL, 312, 69), "a spilled chunk stayed"); + STD_ASSERT(313, 13, txn_blobs_tests_spilled(io, &deltas, &chunk->f0, &bytes, 313, 70) && + bytes.len == chunk->f1.len && memcmp(bytes.ptr, chunk->f1.ptr, bytes.len) == 0, + "spill::chunk(io, deltas.record(), &address.0).unwrap() == Some(bytes)"); + std_vec_free_Checked(&bytes, 1, 1); + } + txn_blobs_tests_ended(io, false, 316, 37); + for (size_t k = 0; k < chunks.len; k++) + STD_ASSERT_MSG(318, 13, !txn_blobs_tests_spilled(io, &deltas, &STD_VEC_AT(&chunks, txn_TupleAddressVecU8_t, + k).f0, NULL, 318, 70), + "an aborted spill's chunk read"); + txn_blobs_tests_chunks_drop(&chunks); + host_shared_Held_drop(&deltas); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* txn_blobs_tests_holding_t is the holding test's actors' captures: the io, the deltas, and the end each actor's turn makes. */ +typedef struct txn_blobs_tests_holding_t +{ + HostIoIo *io; + const HostSharedHeld *deltas; + bool committed; +} txn_blobs_tests_holding_t; + +/* txn_blobs_tests_holding_write is an actor's turn that writes the holding test's chunk under its own transaction. */ +static void +txn_blobs_tests_holding_write(void *captures) +{ + txn_blobs_tests_holding_t *c = captures; + + txn_blobs_tests_written(c->io, c->deltas, 9325, txn_blobs_tests_current(c->io), txn_blobs_tests_chunked(7, 8), + true, 334, 95); +} + +/* txn_blobs_tests_holding_end is an actor's turn that ends its transaction as its captures say. */ +static void +txn_blobs_tests_holding_end(void *captures) +{ + txn_blobs_tests_holding_t *c = captures; + + txn_blobs_tests_ended(c->io, c->committed, c->committed ? 345 : 340, c->committed ? 39 : 41); +} + +/* + * test_txn_blobs_tests_a_chunk_another_running_transaction_holds_stays_after_that_ones_abort: + * PLAN's chunk spill row, the writer's rule, a transaction writing a chunk + * another running transaction holds pushes its own version of it, so the + * other's abort, which takes its versions back, leaves the chunk the first + * one names. + */ +void +test_txn_blobs_tests_a_chunk_another_running_transaction_holds_stays_after_that_ones_abort(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld deltas = txn_blobs_tests_deltas(io, 9325, 330, 73); + StdVec chunks = txn_blobs_tests_chunked(7, 8); + txn_TupleAddressVecU8_t *chunk = &STD_VEC_AT(&chunks, txn_TupleAddressVecU8_t, 0); + txn_blobs_tests_holding_t captures = {.io = io, .deltas = &deltas}; + StdVec bytes; + + simc_as_actor(1, txn_blobs_tests_holding_write, &captures); + simc_as_actor(2, txn_blobs_tests_holding_write, &captures); + captures.committed = false; + simc_as_actor(1, txn_blobs_tests_holding_end, &captures); + STD_ASSERT(342, 9, txn_blobs_tests_spilled(io, &deltas, &chunk->f0, &bytes, 342, 66) && + bytes.len == chunk->f1.len && memcmp(bytes.ptr, chunk->f1.ptr, bytes.len) == 0, + "spill::chunk(io, deltas.record(), &address.0).unwrap() == Some(bytes)"); + std_vec_free_Checked(&bytes, 1, 1); + captures.committed = true; + simc_as_actor(2, txn_blobs_tests_holding_end, &captures); + txn_blobs_tests_chunks_drop(&chunks); + host_shared_Held_drop(&deltas); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * test_txn_blobs_tests_a_rolled_back_chunk_takes_nothing_from_its_tops_count: + * PLAN's chunk spill row, a chunk's version counts at no status entry, so a + * subtransaction's rollback, which prunes its chunk versions, leaves its + * top-level transaction's count as its rows left it. + */ +void +test_txn_blobs_tests_a_rolled_back_chunk_takes_nothing_from_its_tops_count(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld deltas = txn_blobs_tests_deltas(io, 9326, 357, 73); + HostXid top = {txn_blobs_tests_current(io).xid}; + TxnMemtableVersionWriter sub; + TxnStatusRoomed roomed; + TxnStatusEntry entry; + StdVec chunks; + uint64 added; + bool has; + uint64 got; + uint64 want = 1; + HostError error; + + txn_blobs_tests_ok(txn_status_roomed(io, top, &roomed, &error), &error, 359, 45); + txn_blobs_tests_ok(txn_status_add(io, top, 1, &has, &added, &error), &error, 360, 45); + simc_savepoint(); + sub = txn_blobs_tests_current(io); + txn_blobs_tests_ok(txn_written_sub_started(io, (uint32) sub.xid, &error), &error, 363, 57); + txn_blobs_tests_written(io, &deltas, 9326, sub, txn_blobs_tests_chunked(3, 5), true, 364, 88); + simc_rollback_to(); + txn_blobs_tests_ok(txn_written_sub_aborted(io, (uint32) sub.xid, &error), &error, 366, 57); + chunks = txn_blobs_tests_chunked(3, 4); + STD_ASSERT(367, 9, !txn_blobs_tests_spilled(io, &deltas, &STD_VEC_AT(&chunks, txn_TupleAddressVecU8_t, 0).f0, NULL, + 367, 77), + "spill::chunk(io, deltas.record(), &chunked(3..4)[0].0.0).unwrap() == None"); + txn_blobs_tests_chunks_drop(&chunks); + txn_blobs_tests_ok(txn_status_lookup(io, top, &has, &entry, &error), &error, 368, 56); + got = has ? entry.count : 0; + STD_ASSERT_EQ(368, 9, has && entry.count == 1, &got, std_u64_Debug_fmt, &want, std_u64_Debug_fmt); + txn_blobs_tests_ended(io, true, 370, 35); + host_shared_Held_drop(&deltas); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +#endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/change.c b/src/doltgres/txn/c/change.c index a684d9c2136fa..556263c694961 100644 --- a/src/doltgres/txn/c/change.c +++ b/src/doltgres/txn/c/change.c @@ -38,6 +38,16 @@ #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/repo/lib.h" +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" +#include "doltgres/txn/insert.h" +#include "doltgres/txn/state.h" +#endif + +#define RUST_FILE "txn" STD_SEP "src" STD_SEP "change.rs" + /* The prepare whose closure answers an end, or none to start over. */ #define TXN_MEMTABLE_TABLE_RS OptionEnded #define TXN_MEMTABLE_TABLE_RETURNS bool @@ -484,3 +494,491 @@ txn_change_redo(HostIoIo *io, const HostSharedHeld *held, const TxnRecordRowChan return txn_memtable_table_prepare_Handle(io, record, &rows, 1, NULL, 0, false, txn_change_redo_closure_0, &inside, NULL, &handle, error); } + +#ifdef DOLTGRES_TEST_TXN + +/* + * test_txn_change_tests_a_keys_stripe_is_its_storage_and_bytes_or_its_storage_alone: + * a key's stripe of the key lock is the same every time and below the + * stripes' count; two storages' stripes of one key differ, as do two keys' + * of one storage, for the keys this case picks; and a storage whose equal + * keys can have other bytes takes one stripe for every key. + */ +void +test_txn_change_tests_a_keys_stripe_is_its_storage_and_bytes_or_its_storage_alone(void) +{ + HostStorage one = {.tablespace = HOST_PG_DEFAULT, .database = 5, .relation = 16384}; + HostStorage two = {.tablespace = HOST_PG_DEFAULT, .database = 5, .relation = 16385}; + const uint8 *k = (const uint8 *) "k"; + const uint8 *l = (const uint8 *) "l"; + + STD_ASSERT(__LINE__, 9, txn_change_stripe(&one, k, 1, true) == txn_change_stripe(&one, k, 1, true), + "a key's stripe is the same every time"); + STD_ASSERT(__LINE__, 9, txn_change_stripe(&one, k, 1, true) < HOST_STRIPES, "a key's stripe is below the count"); + STD_ASSERT(__LINE__, 9, txn_change_stripe(&one, k, 1, true) != txn_change_stripe(&two, k, 1, true), + "two storages' stripes of one key differ"); + STD_ASSERT(__LINE__, 9, txn_change_stripe(&one, k, 1, true) != txn_change_stripe(&one, l, 1, true), + "two keys' stripes of one storage differ"); + STD_ASSERT(__LINE__, 9, txn_change_stripe(&one, k, 1, false) == txn_change_stripe(&one, l, 1, false), + "a storage whose equal keys can differ in bytes takes one stripe for every key"); +} + +/* TXN_CHANGE_TESTS_UNWRAP_FAILED is a Result's unwrap's panic on a HostError at a line and column. */ +#define TXN_CHANGE_TESTS_UNWRAP_FAILED(line, col, error) \ + STD_RESULT_EXPECT_FAILED((line), (col), "called `Result::unwrap()` on an `Err` value", (error), \ + host_Error_Debug_fmt) + +/* txn_change_tests_made_t is made's (Storage, Repo, Held) with the reader its store reads through. */ +typedef struct txn_change_tests_made_t +{ + HostStorage storage; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; +} txn_change_tests_made_t; + +/* txn_change_tests_made returns a new database's storage in pg_default with its repository, a reader, and its state. */ +static txn_change_tests_made_t +txn_change_tests_made(HostIoIo *io, uint32 database) +{ + txn_change_tests_made_t made = {.storage = txn_insert_tests_stored(database)}; + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_create(io, database, HOST_PG_DEFAULT, &made.repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!host_shared_read(io, &made.reading, &error) || + !txn_state_state(io, &made.repo, &made.reading, &made.storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, + &made.held, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 0, &error); + return made; +} + +/* txn_change_tests_made_drop frees what made returned. */ +static void +txn_change_tests_made_drop(txn_change_tests_made_t *made) +{ + host_shared_Held_drop(&made->held); + host_shared_Reader_drop(&made->reading); + repo_Repo_drop(&made->repo); +} + +/* txn_change_tests_logged_t is one of changes' row changes with its record's end. */ +typedef struct txn_change_tests_logged_t +{ + TxnRecordRowChange change; + uint64 end; +} txn_change_tests_logged_t; + +/* txn_change_tests_changes returns a database's storages' logged row changes, each with its record's end. */ +static StdVec +txn_change_tests_changes(uint32 database) +{ + StdVec logged = simc_logged(); + StdVec changes; + + std_vec_new(&changes, sizeof(txn_change_tests_logged_t), _Alignof(txn_change_tests_logged_t)); + for (size_t k = 0; k < logged.len; k++) + { + simc_TupleU8VecU8Lsn_t *one = &STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, k); + txn_change_tests_logged_t row = {.end = one->f2}; + HostWalKind kind; + HostError error; + + if (host_WalKind_of(one->f0, &kind) && kind == HOST_WAL_KIND_ROW_CHANGE) + { + if (!txn_record_RowChange_decode(one->f1.ptr, one->f1.len, &row.change, &error)) + host_Error_drop(&error); + else if (row.change.storage.database != database) + txn_record_RowChange_drop(&row.change); + else + std_vec_push_Checked(&changes, &row, sizeof(row), _Alignof(txn_change_tests_logged_t)); + } + std_vec_free_Checked(&one->f1, 1, 1); + } + std_vec_free_Checked(&logged, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + return changes; +} + +/* txn_change_tests_changes_drop frees what changes returned. */ +static void +txn_change_tests_changes_drop(StdVec *changes) +{ + for (size_t k = 0; k < changes->len; k++) + txn_record_RowChange_drop(&STD_VEC_AT(changes, txn_change_tests_logged_t, k).change); + std_vec_free_Checked(changes, sizeof(txn_change_tests_logged_t), _Alignof(txn_change_tests_logged_t)); +} + +/* txn_change_tests_shown_t is shown's (String, u32, Option): what a version does, its command, and its LSN. */ +typedef struct txn_change_tests_shown_t +{ + const char *does; + uint32 raw; + bool has_lsn; + uint64 lsn; +} txn_change_tests_shown_t; + +/* txn_change_tests_shown returns key k's versions newest first, each as what it does, its command, and its LSN. */ +static StdVec +txn_change_tests_shown(HostIoIo *io, const HostSharedHeld *held) +{ + HostDeltaHold hold; + StdVec versions; + StdVec shown; + HostError error; + + if (!host_delta_hold(io, &hold, &error) || + !txn_memtable_table_versions(io, &hold, host_shared_Held_record(held), (const uint8 *) "k", 1, &versions, + &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 0, &error); + std_vec_new(&shown, sizeof(txn_change_tests_shown_t), _Alignof(txn_change_tests_shown_t)); + for (size_t k = 0; k < versions.len; k++) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(&versions, + TxnMemtableVersionVersion, + k)); + TxnMemtableVersionOpRefTag op = txn_memtable_version_Record_op(&record).tag; + txn_change_tests_shown_t one = {.does = "other", .raw = txn_memtable_version_Record_raw(&record)}; + + if (op == TXN_MEMTABLE_VERSION_OP_REF_PUT) + one.does = "put"; + else if (op == TXN_MEMTABLE_VERSION_OP_REF_DELETE) + one.does = "delete"; + else if (op == TXN_MEMTABLE_VERSION_OP_REF_LOCK) + one.does = "lock"; + one.has_lsn = txn_memtable_version_Record_lsn(&record, &one.lsn); + std_vec_push_Checked(&shown, &one, sizeof(one), _Alignof(txn_change_tests_shown_t)); + } + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + return shown; +} + +/* txn_change_tests_shown_eq reports whether two shown versions are the same. */ +static bool +txn_change_tests_shown_eq(const txn_change_tests_shown_t *a, const txn_change_tests_shown_t *b) +{ + return strcmp(a->does, b->does) == 0 && a->raw == b->raw && a->has_lsn == b->has_lsn && + (!a->has_lsn || a->lsn == b->lsn); +} + +/* txn_change_tests_shown_is asserts that key k's versions show as expected, freeing what shown returned. */ +static void +txn_change_tests_shown_is(uint32 line, StdVec *got, const txn_change_tests_shown_t *want, size_t want_len) +{ + bool eq = got->len == want_len; + + for (size_t k = 0; eq && k < want_len; k++) + eq = txn_change_tests_shown_eq(&STD_VEC_AT(got, txn_change_tests_shown_t, k), &want[k]); + STD_ASSERT(line, 9, eq, "a key's versions show as expected"); + std_vec_free_Checked(got, sizeof(txn_change_tests_shown_t), _Alignof(txn_change_tests_shown_t)); +} + +/* txn_change_tests_row returns an insert of key k's row by a writer, with its writer's room or none. */ +static TxnInsertInsert +txn_change_tests_row(HostStorage storage, TxnMemtableVersionWriter writer, bool has_roomed, TxnStatusRoomed roomed) +{ + TxnInsertInsert row = txn_insert_tests_row(storage, 0); + + std_vec_free_Checked(&row.key, 1, 1); + std_slice_to_vec_Checked(&row.key, (const uint8 *) "k", 1, 1, 1); + row.writer = writer; + row.has_roomed = has_roomed; + row.roomed = roomed; + return row; +} + +/* txn_change_tests_inserted inserts key k's row and returns its tuple id. */ +static uint64 +txn_change_tests_inserted(HostIoIo *io, const txn_change_tests_made_t *made, TxnInsertInsert row) +{ + TreesStoreReader store = repo_Repo_reader(&made->repo, HOST_CACHE_PLACEMENT_SERVER, &made->reading); + TxnInsertRoom room = {.bytes = 28, .page = 8168, .most = 291}; + uint64 tid; + HostError error; + + if (!txn_insert_tests_taken(io, TREES_STORE_READER_NODE_STORE(&store), &made->held, &row, &room, NULL, NULL, 0, + &tid, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 0, &error); + return tid; +} + +/* txn_change_tests_ending returns the end of key k's row of a tuple id by a writer at a command, doing an op. */ +static TxnChangeEnding +txn_change_tests_ending(HostStorage storage, TxnMemtableOp op, TxnMemtableVersionWriter writer, uint32 command, + TxnStatusRoomed roomed, uint64 tid) +{ + TxnChangeEnding ending = {.storage = storage, .table = 16385, .op = op, .writer = writer, .roomed = roomed, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .logged = true, .tuple = {tid}}; + + ending.writer.command = command; + std_slice_to_vec_Checked(&ending.key, (const uint8 *) "k", 1, 1, 1); + return ending; +} + +/* txn_change_tests_stop is a read that stops with the count of the versions it read, its Stop's usize. */ +static bool +txn_change_tests_stop(void *captures, HostIoIo *io, TxnMemtableVersionVersion *versions, size_t versions_len, + TxnChangeRead *result, void *stopped, HostError *error) +{ + result->tag = TXN_CHANGE_READ_STOP; + *(size_t *) stopped = versions_len; + return true; +} + +/* txn_change_tests_go is a read that goes on to the push, its version showing its writer's command. */ +static bool +txn_change_tests_go(void *captures, HostIoIo *io, TxnMemtableVersionVersion *versions, size_t versions_len, + TxnChangeRead *result, void *stopped, HostError *error) +{ + result->tag = TXN_CHANGE_READ_GO; + result->u.go.shown.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND; + result->u.go.cmin = 0; + result->u.go.has_row = false; + return true; +} + +/* txn_change_tests_ended ends a row with a read and returns whether it pushed, a Stop's count in stopped. */ +static TxnChangeEnded +txn_change_tests_ended(HostIoIo *io, const HostSharedHeld *held, TxnChangeEnding ending, TxnChangeDecide decide, + size_t *stopped) +{ + HostIoWaits waits = host_io_Waits_testing(); + uint32 stripe = txn_change_stripe(&ending.storage, (const uint8 *) "k", 1, true); + TxnChangeEnded ended; + HostError error; + + if (!txn_change_end(io, held, &ending, &waits, stripe, decide, NULL, NULL, stopped, &ended, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 0, &error); + return ended; +} + +/* txn_change_tests_op_delete returns a delete's op. */ +static TxnMemtableOp +txn_change_tests_op_delete(void) +{ + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_DELETE}; + + return op; +} + +/* txn_change_tests_op_share returns a share lock's op. */ +static TxnMemtableOp +txn_change_tests_op_share(void) +{ + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_LOCK, .u.lock = TXN_MEMTABLE_LOCK_MODE_SHARE}; + + return op; +} + +/* + * test_txn_change_tests_a_delete_is_logged_pushed_and_redone_once_and_a_lock_is_never_logged: + * a delete whose read stops, seeing the key's one version, pushes nothing and + * gives back what it stopped with, and a logged delete of an inserted row + * appends its row change and pushes its version with the record's LSN, + * showing its own command; redone on another storage, the insert and the + * delete each show command 0, as heap's redo writes FirstCommandId, and a + * delete redone again, which the delta holds by its key and LSN, pushes + * nothing. A lock pushes its version with no record. + */ +void +test_txn_change_tests_a_delete_is_logged_pushed_and_redone_once_and_a_lock_is_never_logged(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_change_tests_made_t made = txn_change_tests_made(io, 9361); + TxnMemtableVersionWriter writer = {.xid = 41, .command = 2}; + bool has_roomed; + TxnStatusRoomed roomed; + uint64 tid; + size_t stopped = 0; + StdVec logged; + const txn_change_tests_logged_t *inserted; + const txn_change_tests_logged_t *deleted; + txn_change_tests_made_t replica; + TreesStoreReader replica_store; + TxnInsertPage page = {.bytes = 8168, .most = 291}; + HostError error; + + if (!txn_status_redo_roomed(io, &writer, &has_roomed, &roomed, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + STD_ASSERT(__LINE__, 9, has_roomed, "the writer takes room"); + tid = txn_change_tests_inserted(io, &made, txn_change_tests_row(made.storage, writer, true, roomed)); + STD_ASSERT(__LINE__, 9, + txn_change_tests_ended(io, &made.held, + txn_change_tests_ending(made.storage, txn_change_tests_op_delete(), writer, 3, + roomed, tid), + txn_change_tests_stop, &stopped) == TXN_CHANGE_ENDED_STOPPED && stopped == 1, + "a delete whose read stops pushes nothing and gives back what it stopped with"); + STD_ASSERT(__LINE__, 9, + txn_change_tests_ended(io, &made.held, + txn_change_tests_ending(made.storage, txn_change_tests_op_delete(), writer, 3, + roomed, tid), + txn_change_tests_go, &stopped) == TXN_CHANGE_ENDED_PUSHED, + "a delete whose read goes on pushes its version"); + logged = txn_change_tests_changes(9361); + STD_ASSERT(__LINE__, 9, logged.len == 2, "the insert and the delete each logged a row change"); + inserted = &STD_VEC_AT(&logged, txn_change_tests_logged_t, 0); + deleted = &STD_VEC_AT(&logged, txn_change_tests_logged_t, 1); + STD_ASSERT(__LINE__, 9, deleted->change.op.tag == TXN_MEMTABLE_OP_DELETE, "the second row change is the delete"); + { + txn_change_tests_shown_t want[] = {{"delete", 3, true, deleted->end}, {"put", 2, true, inserted->end}}; + StdVec got = txn_change_tests_shown(io, &made.held); + + txn_change_tests_shown_is(__LINE__, &got, want, lengthof(want)); + } + replica = txn_change_tests_made(io, 9362); + replica_store = repo_Repo_reader(&replica.repo, HOST_CACHE_PLACEMENT_SERVER, &replica.reading); + for (size_t k = 0; k < logged.len; k++) + { + const txn_change_tests_logged_t *one = &STD_VEC_AT(&logged, txn_change_tests_logged_t, k); + const TxnMemtablePut *put; + bool ok; + + if (txn_record_RowChange_put(&one->change, &put)) + ok = txn_insert_redo(io, TREES_STORE_READER_NODE_STORE(&replica_store), &replica.held, &one->change, + one->end, &page, NULL, NULL, 0, &error); + else + ok = txn_change_redo(io, &replica.held, &one->change, one->end, &error); + if (!ok) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 13, &error); + } + if (!txn_change_redo(io, &replica.held, &deleted->change, deleted->end, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + { + txn_change_tests_shown_t want[] = {{"delete", 0, true, deleted->end}, {"put", 0, true, inserted->end}}; + StdVec got = txn_change_tests_shown(io, &replica.held); + + txn_change_tests_shown_is(__LINE__, &got, want, lengthof(want)); + } + STD_ASSERT(__LINE__, 9, + txn_change_tests_ended(io, &made.held, + txn_change_tests_ending(made.storage, txn_change_tests_op_share(), writer, 4, + roomed, tid), + txn_change_tests_go, &stopped) == TXN_CHANGE_ENDED_PUSHED, + "a lock pushes its version"); + { + StdVec got = txn_change_tests_shown(io, &made.held); + txn_change_tests_shown_t lock = {"lock", 4, false, 0}; + + STD_ASSERT(__LINE__, 9, got.len > 0 && + txn_change_tests_shown_eq(&STD_VEC_AT(&got, txn_change_tests_shown_t, 0), &lock), + "the key's newest version is the lock, with no LSN"); + std_vec_free_Checked(&got, sizeof(txn_change_tests_shown_t), _Alignof(txn_change_tests_shown_t)); + } + { + StdVec again = txn_change_tests_changes(9361); + + STD_ASSERT(__LINE__, 9, again.len == 2, "a lock logs no row change"); + txn_change_tests_changes_drop(&again); + } + if (!txn_written_ended(io, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 21, &error); + txn_change_tests_changes_drop(&logged); + txn_change_tests_made_drop(&replica); + txn_change_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_change_tests_an_aborted_delete_and_lock_go_at_the_take_back: an + * aborted transaction's delete and lock of a committed row go at its + * take-back, since end records the key in its list before its prepare, and + * the committed row's put stays the key's one version. + */ +void +test_txn_change_tests_an_aborted_delete_and_lock_go_at_the_take_back(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_change_tests_made_t made = txn_change_tests_made(io, 9363); + HostXid committed = simc_begin(); + HostXid aborted = simc_begin(); + TxnMemtableVersionWriter by_committed = {.xid = committed.f0}; + TxnMemtableVersionWriter by_aborted = {.xid = aborted.f0}; + TxnStatusRoomed unroomed = {0}; + TxnStatusRoomed roomed; + uint64 tid; + StdVec got; + HostError error; + + tid = txn_change_tests_inserted(io, &made, txn_change_tests_row(made.storage, by_committed, false, unroomed)); + simc_commit(committed); + if (!txn_written_ended(io, &error) || !txn_status_roomed(io, aborted, &roomed, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + txn_change_tests_ended(io, &made.held, + txn_change_tests_ending(made.storage, txn_change_tests_op_share(), by_aborted, 0, roomed, + tid), + txn_change_tests_go, NULL); + txn_change_tests_ended(io, &made.held, + txn_change_tests_ending(made.storage, txn_change_tests_op_delete(), by_aborted, 0, roomed, + tid), + txn_change_tests_go, NULL); + got = txn_change_tests_shown(io, &made.held); + STD_ASSERT(__LINE__, 9, got.len == 3, "the key holds the put, the lock, and the delete"); + std_vec_free_Checked(&got, sizeof(txn_change_tests_shown_t), _Alignof(txn_change_tests_shown_t)); + simc_abort(aborted); + if (!txn_written_aborted(io, 0, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + got = txn_change_tests_shown(io, &made.held); + STD_ASSERT(__LINE__, 9, got.len == 1 && strcmp(STD_VEC_AT(&got, txn_change_tests_shown_t, 0).does, "put") == 0, + "the take-back leaves the committed row's put the key's one version"); + std_vec_free_Checked(&got, sizeof(txn_change_tests_shown_t), _Alignof(txn_change_tests_shown_t)); + txn_change_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_change_tests_a_change_of_a_spilled_row_reads_its_spilled_version: + * a change of a row whose version its transaction spilled reads that version + * from the spill, though the row's slice of the memtable's chain holds none of + * the key's versions, and pushes its own version over it. + */ +void +test_txn_change_tests_a_change_of_a_spilled_row_reads_its_spilled_version(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_change_tests_made_t made = txn_change_tests_made(io, 9524); + HostXid xid = simc_begin(); + TxnMemtableVersionWriter writer = {.xid = xid.f0, .command = 2}; + TxnStatusRoomed roomed; + uint64 tid; + bool spilled; + size_t stopped = 0; + StdVec got; + HostError error; + + if (!txn_status_roomed(io, xid, &roomed, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + tid = txn_change_tests_inserted(io, &made, txn_change_tests_row(made.storage, writer, true, roomed)); + if (!txn_spill_spilled(io, &made.storage, TXN_WRITTEN_OF_ROWS, writer.xid, &spilled, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + STD_ASSERT(__LINE__, 5, spilled, "the insert spilled its row"); + STD_ASSERT(__LINE__, 9, + txn_change_tests_ended(io, &made.held, + txn_change_tests_ending(made.storage, txn_change_tests_op_delete(), writer, 3, + roomed, tid), + txn_change_tests_stop, &stopped) == TXN_CHANGE_ENDED_STOPPED && stopped == 1, + "the change reads the row's spilled version"); + STD_ASSERT(__LINE__, 9, + txn_change_tests_ended(io, &made.held, + txn_change_tests_ending(made.storage, txn_change_tests_op_delete(), writer, 3, + roomed, tid), + txn_change_tests_go, &stopped) == TXN_CHANGE_ENDED_PUSHED, + "the change pushes its own version over it"); + got = txn_change_tests_shown(io, &made.held); + STD_ASSERT(__LINE__, 9, got.len == 2 && strcmp(STD_VEC_AT(&got, txn_change_tests_shown_t, 0).does, "delete") == 0 && + strcmp(STD_VEC_AT(&got, txn_change_tests_shown_t, 1).does, "put") == 0, + "the key's versions are the delete over the put"); + std_vec_free_Checked(&got, sizeof(txn_change_tests_shown_t), _Alignof(txn_change_tests_shown_t)); + if (!txn_written_ended(io, &error)) + TXN_CHANGE_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + txn_change_tests_made_drop(&made); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +#endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/entry.c b/src/doltgres/txn/c/entry.c index 28c2ae9e2097d..17bb308787092 100644 --- a/src/doltgres/txn/c/entry.c +++ b/src/doltgres/txn/c/entry.c @@ -39,6 +39,18 @@ #include "doltgres/txn/visible.h" #include "doltgres/txn/written.h" +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/repo/lib.h" +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" +#include "doltgres/txn/change.h" +#include "doltgres/txn/record.h" +#include "doltgres/txn/runs.h" + +/* RUST_FILE is the file the tests' asserts name, entry.rs. */ +#define RUST_FILE "txn" STD_SEP "src" STD_SEP "entry.rs" +#endif + /* The prepare whose closure pushes an entry's changes. */ #define TXN_MEMTABLE_TABLE_RS Unit #define TXN_MEMTABLE_TABLE_RETURNS bool @@ -1237,3 +1249,673 @@ txn_entry_unique(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld * txn_entry_unique_pad(&frame); return frame.ok; } + +#ifdef DOLTGRES_TEST_TXN +/* TXN_ENTRY_TESTS_UNWRAP is Result::unwrap's panic text. */ +#define TXN_ENTRY_TESTS_UNWRAP "called `Result::unwrap()` on an `Err` value" + +/* txn_entry_tests_ok unwraps a host result at a line. */ +static void +txn_entry_tests_ok(bool ok, HostError *error, uint32 line) +{ + if (!ok) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_ENTRY_TESTS_UNWRAP, error, host_Error_Debug_fmt); +} + +/* txn_entry_tests_made_t is made's (Storage, Repo, Held), with the reader the state was made under. */ +typedef struct txn_entry_tests_made_t +{ + HostStorage storage; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; +} txn_entry_tests_made_t; + +/* txn_entry_tests_made returns a new database's index storage in pg_default, its state made as a backend's make of an index's storage makes it. */ +static txn_entry_tests_made_t +txn_entry_tests_made(HostIoIo *io, uint32 database) +{ + txn_entry_tests_made_t made = {.storage = {HOST_PG_DEFAULT, database, 16390}}; + TxnStateMade state = {.key = {.tag = TXN_BASE_KEY_NONE}, .table = 16385, .has_index = true, .index = 16390}; + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_create(io, database, HOST_PG_DEFAULT, &made.repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, TXN_ENTRY_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + txn_entry_tests_ok(host_shared_read(io, &made.reading, &error) && + txn_state_state(io, &made.repo, &made.reading, &made.storage, TXN_STATE_PERSISTENCE_PERMANENT, + true, &state, &made.held, &error), &error, __LINE__); + return made; +} + +/* txn_entry_tests_made_drop frees what made returned. */ +static void +txn_entry_tests_made_drop(txn_entry_tests_made_t *made) +{ + host_shared_Held_drop(&made->held); + host_shared_Reader_drop(&made->reading); + repo_Repo_drop(&made->repo); +} + +/* txn_entry_tests_shown_t is shown's (&str, u64, u32, Option): what a version does, its tuple id, its command, and its LSN. */ +typedef struct txn_entry_tests_shown_t +{ + const char *does; + uint64 tuple; + uint32 raw; + bool has_lsn; + uint64 lsn; +} txn_entry_tests_shown_t; + +/* txn_entry_tests_shown returns a key's versions newest first, each as what it does, its tuple id, its command, and its LSN. */ +static StdVec +txn_entry_tests_shown(HostIoIo *io, const HostSharedHeld *held, const uint8 *key, size_t key_len) +{ + HostDeltaHold hold; + StdVec versions; + StdVec shown; + HostError error; + + txn_entry_tests_ok(host_delta_hold(io, &hold, &error) && + txn_memtable_table_versions(io, &hold, host_shared_Held_record(held), key, key_len, &versions, + &error), &error, __LINE__); + std_vec_new(&shown, sizeof(txn_entry_tests_shown_t), _Alignof(txn_entry_tests_shown_t)); + for (size_t k = 0; k < versions.len; k++) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(&versions, + TxnMemtableVersionVersion, + k)); + TxnMemtableVersionOpRef op = txn_memtable_version_Record_op(&record); + txn_entry_tests_shown_t one = {.does = "other", .raw = txn_memtable_version_Record_raw(&record)}; + + if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_PUT) + { + one.does = "put"; + one.tuple = op.u.put.tuple.f0; + } + else if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_DELETE) + one.does = "delete"; + else if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_ADD) + { + one.does = "add"; + one.tuple = op.u.add.tuple.f0; + } + else if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_REMOVE) + { + one.does = "remove"; + one.tuple = op.u.remove.f0.f0; + } + one.has_lsn = txn_memtable_version_Record_lsn(&record, &one.lsn); + std_vec_push_Checked(&shown, &one, sizeof(one), _Alignof(txn_entry_tests_shown_t)); + } + for (size_t k = 0; k < versions.len; k++) + txn_memtable_version_Version_drop(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, k)); + std_vec_free_Checked(&versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + host_delta_Hold_drop(&hold); + return shown; +} + +/* txn_entry_tests_shown_Debug_fmt is a shown version's Debug. */ +static bool +txn_entry_tests_shown_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_entry_tests_shown_t *one = value; + StdStr does = {one->does, strlen(one->does)}; + StdOptionRef lsn = {one->has_lsn, STD_ITEM(&one->lsn, std_u64_Debug_fmt)}; + StdTupleRef tuple = STD_TUPLE(4, STD_ITEM(&does, std_str_Debug_fmt), STD_ITEM(&one->tuple, std_u64_Debug_fmt), + STD_ITEM(&one->raw, std_u32_Debug_fmt), STD_ITEM(&lsn, std_option_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_entry_tests_shown_list_Debug_fmt is a Vec of shown versions' Debug. */ +static bool +txn_entry_tests_shown_list_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *shown = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < shown->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(shown, txn_entry_tests_shown_t, k), txn_entry_tests_shown_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_entry_tests_shown_is asserts a key's shown versions are the wanted ones, freeing them. */ +static void +txn_entry_tests_shown_is(uint32 line, StdVec *got, const txn_entry_tests_shown_t *want, size_t want_len) +{ + StdVec wanted = {(void *) want, want_len, want_len}; + bool eq = got->len == want_len; + + for (size_t k = 0; k < want_len && eq; k++) + { + const txn_entry_tests_shown_t *a = &STD_VEC_AT(got, txn_entry_tests_shown_t, k); + + eq = strcmp(a->does, want[k].does) == 0 && a->tuple == want[k].tuple && a->raw == want[k].raw && + a->has_lsn == want[k].has_lsn && (!a->has_lsn || a->lsn == want[k].lsn); + } + STD_ASSERT_EQ(line, 9, eq, got, txn_entry_tests_shown_list_Debug_fmt, &wanted, + txn_entry_tests_shown_list_Debug_fmt); + std_vec_free_Checked(got, sizeof(txn_entry_tests_shown_t), _Alignof(txn_entry_tests_shown_t)); +} + +/* txn_entry_tests_does_is asserts what a key's versions do, newest first, freeing them. */ +static void +txn_entry_tests_does_is(uint32 line, StdVec *got, const char *const *want, size_t want_len) +{ + bool eq = got->len == want_len; + + for (size_t k = 0; k < want_len && eq; k++) + eq = strcmp(STD_VEC_AT(got, txn_entry_tests_shown_t, k).does, want[k]) == 0; + STD_ASSERT_MSG(line, 9, eq, "a key's versions do what the test wants"); + std_vec_free_Checked(got, sizeof(txn_entry_tests_shown_t), _Alignof(txn_entry_tests_shown_t)); +} + +/* txn_entry_tests_put returns an op that puts a value under a tuple id. */ +static TxnMemtableOp +txn_entry_tests_put(const char *value, uint64 tuple) +{ + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_PUT}; + + std_slice_to_vec_Checked(&op.u.put.value, value, strlen(value), 1, 1); + std_vec_new(&op.u.put.addresses, sizeof(chunks_TupleU32Locator_t), _Alignof(chunks_TupleU32Locator_t)); + std_vec_new(&op.u.put.values, sizeof(uint32), _Alignof(uint32)); + op.u.put.tuple.f0 = tuple; + return op; +} + +/* txn_entry_tests_entry returns an entry of a storage's index under a writer, its key's change, logged. */ +static TxnEntryEntry +txn_entry_tests_entry(HostIoIo *io, const HostStorage *storage, TxnMemtableVersionWriter writer, uint32 index, + const char *key, TxnMemtableOp op) +{ + TxnEntryEntry entry = {.writer = writer, .storage = *storage, .table = 16385, .index = index, .op = op, + .logged = true}; + bool has; + HostError error; + + txn_entry_tests_ok(txn_status_redo_roomed(io, &writer, &has, &entry.roomed, &error), &error, __LINE__); + STD_ASSERT_MSG(__LINE__, 0, has, "a roomed entry"); + std_slice_to_vec_Checked(&entry.key, key, strlen(key), 1, 1); + return entry; +} + +/* txn_entry_tests_changed changes an entry, which unwraps at a line. */ +static void +txn_entry_tests_changed(HostIoIo *io, const HostSharedHeld *held, TxnEntryEntry entry, uint32 line) +{ + HostError error; + + txn_entry_tests_ok(txn_entry_change(io, held, &entry, false, NULL, &error), &error, line); +} + +/* txn_entry_tests_writer returns a writer of a transaction the simulation begins. */ +static TxnMemtableVersionWriter +txn_entry_tests_writer(HostXid *xid) +{ + TxnMemtableVersionWriter writer = {0}; + + *xid = simc_begin(); + writer.xid = xid->f0; + return writer; +} + +/* txn_entry_tests_logged_t is one of the logged row changes with its record's end. */ +typedef struct txn_entry_tests_logged_t +{ + TxnRecordRowChange change; + uint64 end; +} txn_entry_tests_logged_t; + +/* txn_entry_tests_changes returns a database's logged row changes, each with its record's end. */ +static StdVec +txn_entry_tests_changes(uint32 database) +{ + StdVec logged = simc_logged(); + StdVec changes; + + std_vec_new(&changes, sizeof(txn_entry_tests_logged_t), _Alignof(txn_entry_tests_logged_t)); + for (size_t k = 0; k < logged.len; k++) + { + simc_TupleU8VecU8Lsn_t *one = &STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, k); + txn_entry_tests_logged_t row = {.end = one->f2}; + HostWalKind kind; + HostError error; + + if (host_WalKind_of(one->f0, &kind) && kind == HOST_WAL_KIND_ROW_CHANGE) + { + if (!txn_record_RowChange_decode(one->f1.ptr, one->f1.len, &row.change, &error)) + host_Error_drop(&error); + else if (row.change.storage.database != database) + txn_record_RowChange_drop(&row.change); + else + std_vec_push_Checked(&changes, &row, sizeof(row), _Alignof(txn_entry_tests_logged_t)); + } + std_vec_free_Checked(&one->f1, 1, 1); + } + std_vec_free_Checked(&logged, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + return changes; +} + +/* + * test_txn_entry_tests_an_index_entrys_changes_are_logged_pushed_and_redone_once: + * the btree text's section 5 and PLAN's row change row, field 31, an index's + * entry put and delete, and a counted entry's added and removed copy, each + * append a row change of the index's storage naming its table and its index + * and push their versions with its LSN, showing their writer's command; redo + * of each on another database's storage, first seen by it, makes its first + * base record with that table and index and pushes each version once, + * showing command 0, however often it is replayed. + */ +void +test_txn_entry_tests_an_index_entrys_changes_are_logged_pushed_and_redone_once(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_entry_tests_made_t made = txn_entry_tests_made(io, 9371); + TxnMemtableVersionWriter writer = {.xid = 51, .command = 2}; + uint64 copy = (UINT64CONST(1) << 16) | 1; + TxnMemtableOp added = txn_entry_tests_put("entry", copy); + TxnMemtableOp removed = {.tag = TXN_MEMTABLE_OP_REMOVE}; + TxnMemtableOp deleted = {.tag = TXN_MEMTABLE_OP_DELETE}; + StdVec logged; + uint64 lsns[4]; + HostStorage replica_storage = {HOST_PG_DEFAULT, 9372, 16390}; + RepoRepo replica_repo; + HostSharedReader reading; + HostSharedHeld replica; + TxnBaseBase base; + bool some = false; + ChunksErrorError failed; + HostError error; + + added.tag = TXN_MEMTABLE_OP_ADD; + removed.u.remove.f0.f0 = copy; + txn_entry_tests_changed(io, &made.held, txn_entry_tests_entry(io, &made.storage, writer, 16390, "p", + txn_entry_tests_put("entry", 0)), __LINE__); + txn_entry_tests_changed(io, &made.held, txn_entry_tests_entry(io, &made.storage, writer, 16390, "p", deleted), + __LINE__); + txn_entry_tests_changed(io, &made.held, txn_entry_tests_entry(io, &made.storage, writer, 16390, "c", added), + __LINE__); + txn_entry_tests_changed(io, &made.held, txn_entry_tests_entry(io, &made.storage, writer, 16390, "c", removed), + __LINE__); + logged = txn_entry_tests_changes(9371); + STD_ASSERT_MSG(__LINE__, 9, logged.len == 4, "logged.len() == 4"); + for (size_t k = 0; k < logged.len; k++) + { + const txn_entry_tests_logged_t *one = &STD_VEC_AT(&logged, txn_entry_tests_logged_t, k); + + STD_ASSERT_MSG(__LINE__, 9, one->change.has_index && one->change.index == 16390 && one->change.table == 16385, + "every change names the index and its table"); + lsns[k] = one->end; + } + { + txn_entry_tests_shown_t p[] = {{"delete", 0, 2, true, lsns[1]}, {"put", 0, 2, true, lsns[0]}}; + txn_entry_tests_shown_t c[] = {{"remove", copy, 2, true, lsns[3]}, {"add", copy, 2, true, lsns[2]}}; + StdVec got = txn_entry_tests_shown(io, &made.held, (const uint8 *) "p", 1); + + txn_entry_tests_shown_is(__LINE__, &got, p, lengthof(p)); + got = txn_entry_tests_shown(io, &made.held, (const uint8 *) "c", 1); + txn_entry_tests_shown_is(__LINE__, &got, c, lengthof(c)); + } + if (!repo_Repo_create(io, 9372, HOST_PG_DEFAULT, &replica_repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, TXN_ENTRY_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + txn_entry_tests_ok(host_shared_read(io, &reading, &error), &error, __LINE__); + for (int pass = 0; pass < 2; pass++) + for (size_t k = 0; k < logged.len; k++) + { + const txn_entry_tests_logged_t *one = &STD_VEC_AT(&logged, txn_entry_tests_logged_t, k); + TxnStateMade state = {.key = {.tag = TXN_BASE_KEY_NONE}, .table = one->change.table, + .has_index = one->change.has_index, .index = one->change.index}; + + txn_entry_tests_ok(txn_state_state(io, &replica_repo, &reading, &replica_storage, + TXN_STATE_PERSISTENCE_PERMANENT, true, &state, &replica, &error) && + txn_change_redo(io, &replica, &one->change, one->end, &error), &error, __LINE__); + host_shared_Held_drop(&replica); + } + txn_entry_tests_ok(txn_state_state(io, &replica_repo, &reading, &replica_storage, + TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &replica, &error) && + txn_state_base_record(io, &replica, &some, &base, &error), &error, __LINE__); + STD_ASSERT_MSG(__LINE__, 9, some && base.table == 16385 && base.has_index && base.index == 16390 && + base.key.tag == TXN_BASE_KEY_NONE, "(base.table, base.index, base.key) == (16385, Some(16390), None)"); + txn_base_Base_drop(&base); + { + txn_entry_tests_shown_t p[] = {{"delete", 0, 0, true, lsns[1]}, {"put", 0, 0, true, lsns[0]}}; + txn_entry_tests_shown_t c[] = {{"remove", copy, 0, true, lsns[3]}, {"add", copy, 0, true, lsns[2]}}; + StdVec got = txn_entry_tests_shown(io, &replica, (const uint8 *) "p", 1); + + txn_entry_tests_shown_is(__LINE__, &got, p, lengthof(p)); + got = txn_entry_tests_shown(io, &replica, (const uint8 *) "c", 1); + txn_entry_tests_shown_is(__LINE__, &got, c, lengthof(c)); + } + txn_entry_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + for (size_t k = 0; k < logged.len; k++) + txn_record_RowChange_drop(&STD_VEC_AT(&logged, txn_entry_tests_logged_t, k).change); + std_vec_free_Checked(&logged, sizeof(txn_entry_tests_logged_t), _Alignof(txn_entry_tests_logged_t)); + host_shared_Held_drop(&replica); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&replica_repo); + txn_entry_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* txn_entry_tests_refused reports whether a base entry's read is refused with XX001. */ +static bool +txn_entry_tests_refused(const uint8 *value, size_t value_len, const ChunksTrailerStamp *stamps, size_t stamps_len, + bool keyless) +{ + StdBytes read; + HostError error; + bool refused; + + if (txn_entry_based(value, value_len, stamps, stamps_len, keyless, &read, &error)) + return false; + refused = memcmp(host_Error_sqlstate(&error), "XX001", 5) == 0; + host_Error_drop(&error); + return refused; +} + +/* + * test_txn_entry_tests_an_index_base_entry_reads_back_and_refuses_a_keyed_stamp_and_an_uneven_count: + * PLAN's format row for an index storage's base leaf entry, a keyed table's + * entry reads back its tuple with no stamp and a keyless one its tuple under + * a count equal to its stamps, and each other form is data corruption: a + * stamp on a keyed table's entry, and a count that differs from a keyless + * one's stamps, above or below. + */ +void +test_txn_entry_tests_an_index_base_entry_reads_back_and_refuses_a_keyed_stamp_and_an_uneven_count(void) +{ + ChunksTrailerStamp stamps[3] = {{7, 1}, {7, 2}, {7, 3}}; + StdVec counted; + StdBytes read; + HostError error; + + txn_entry_tests_ok(txn_entry_based((const uint8 *) "tuple", 5, NULL, 0, false, &read, &error), &error, __LINE__); + STD_ASSERT(__LINE__, 9, read.len == 5 && memcmp(read.ptr, "tuple", 5) == 0, + "based(b\"tuple\", &[], false).unwrap() == b\"tuple\""); + txn_counted_keyless_value(2, (const uint8 *) "tuple", 5, &counted); + txn_entry_tests_ok(txn_entry_based(counted.ptr, counted.len, stamps, 2, true, &read, &error), &error, __LINE__); + STD_ASSERT(__LINE__, 9, read.len == 5 && memcmp(read.ptr, "tuple", 5) == 0, + "based(&counted, &[stamp(1), stamp(2)], true).unwrap() == b\"tuple\""); + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_refused((const uint8 *) "tuple", 5, stamps, 1, false), + "a keyed table's entry with a stamp read back"); + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_refused(counted.ptr, counted.len, stamps, 1, true), + "a count above its stamps read back"); + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_refused(counted.ptr, counted.len, stamps, 3, true), + "a count below its stamps read back"); + std_vec_free_Checked(&counted, 1, 1); +} + +/* + * test_txn_entry_tests_an_index_storages_large_keys_spill_at_the_threshold: + * bugs row 153, an index's 6,000 entries of 1 KB keys, whose versions alone + * are a few dozen bytes each, spill by the spill's threshold alone as an + * entries spill, since a write's counted bytes take its key's too. + */ +void +test_txn_entry_tests_an_index_storages_large_keys_spill_at_the_threshold(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io; + HostIoIo *io; + txn_entry_tests_made_t made; + HostXid xid; + TxnMemtableVersionWriter writer; + txn_TupleVecFileU64_t listed; + bool spilled = false; + HostError error; + + simc_size_delta(UINT64CONST(32) << 20); + test_io = simc_io(); + io = &test_io.io; + made = txn_entry_tests_made(io, 9373); + writer = txn_entry_tests_writer(&xid); + for (uint32 number = 0; number < 6000; number++) + { + TxnEntryEntry entry = txn_entry_tests_entry(io, &made.storage, writer, 16390, "", + txn_entry_tests_put("entry", 0)); + + entry.logged = false; + std_vec_free_Checked(&entry.key, 1, 1); + std_vec_from_elem_zeroed(&entry.key, 1 << 10, 1, 1); + for (int b = 0; b < 4; b++) + ((uint8 *) entry.key.ptr)[b] = (uint8) (number >> (8 * (3 - b))); + txn_entry_tests_changed(io, &made.held, entry, __LINE__); + } + txn_entry_tests_ok(txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, host_shared_Held_record(&made.held), + &made.storage, &listed, &error), &error, __LINE__); + for (size_t k = 0; k < listed.f0.len; k++) + spilled |= STD_VEC_AT(&listed.f0, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_ENTRIES; + STD_ASSERT_MSG(__LINE__, 9, spilled, "the index's entries spilled as an entries spill"); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + txn_entry_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + txn_entry_tests_made_drop(&made); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* txn_entry_tests_unique_t is a unique test's captures: what its entries' unique checks read. */ +typedef struct txn_entry_tests_unique_t +{ + HostIoIo *io; + txn_entry_tests_made_t *made; + TreesStoreReader store; + TxnInsertUniqueness uniqueness; + HostIoWaits waits; +} txn_entry_tests_unique_t; + +/* txn_entry_tests_checked runs a unique index's check of an entry in a transaction it commits, answering what it did. */ +static TxnEntryChanged +txn_entry_tests_checked(txn_entry_tests_unique_t *u, const char *key, TxnMemtableOp op, uint64 tuple, bool partial, + bool commit, HostXid xid) +{ + TxnMemtableVersionWriter writer = {.xid = xid.f0}; + TxnEntryEntry entry = txn_entry_tests_entry(u->io, &u->made->storage, writer, u->uniqueness.index, key, op); + TxnEntryChanged changed; + HostRow row = {tuple}; + HostError error; + + txn_entry_tests_ok(txn_entry_unique(u->io, TREES_STORE_READER_NODE_STORE(&u->store), &u->made->held, &entry, false, + NULL, &u->uniqueness, &u->waits, row, partial, &changed, &error), &error, + __LINE__); + if (commit) + { + simc_commit(xid); + txn_entry_tests_ok(txn_written_ended(u->io, &error), &error, __LINE__); + } + return changed; +} + +/* txn_entry_tests_unique begins a unique test's captures over a made index storage, for an index. */ +static txn_entry_tests_unique_t +txn_entry_tests_unique(HostIoIo *io, txn_entry_tests_made_t *made, const TreesKeyOrder *order, uint32 index) +{ + txn_entry_tests_unique_t u = {.io = io, .made = made, .uniqueness = {.order = order, .index = index}, + .waits = host_io_Waits_testing()}; + + u.store = repo_Repo_reader(&made->repo, HOST_CACHE_PLACEMENT_SERVER, &made->reading); + return u; +} + +/* txn_entry_tests_exists_is reports whether a unique check answered an existing entry of a value, freeing it. */ +static bool +txn_entry_tests_exists_is(TxnEntryChanged changed, const char *value) +{ + bool is = changed.tag == TXN_ENTRY_CHANGED_EXISTS && changed.u.exists.len == strlen(value) && + memcmp(changed.u.exists.ptr, value, changed.u.exists.len) == 0; + + if (changed.tag == TXN_ENTRY_CHANGED_EXISTS) + std_vec_free_Checked(&changed.u.exists, 1, 1); + return is; +} + +/* + * test_txn_entry_tests_a_unique_entry_takes_a_free_key_and_answers_an_existing_entrys_value: + * the btree's part 2, slice 5, a unique index's entry resolves its key as an + * immediate key's insert does: a free key takes the put, a key holding a + * committed entry answers that entry's value and writes nothing, and once a + * committed delete ended the entry, the key takes the put again. + */ +void +test_txn_entry_tests_a_unique_entry_takes_a_free_key_and_answers_an_existing_entrys_value(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_entry_tests_made_t made = txn_entry_tests_made(io, 9373); + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + txn_entry_tests_unique_t u = txn_entry_tests_unique(io, &made, &order, 16390); + static const char *const ops[] = {"put", "delete", "put"}; + HostXid xid; + StdVec shown; + HostError error; + + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_checked(&u, "k", txn_entry_tests_put("a", 0), 1, false, true, + simc_begin()).tag == TXN_ENTRY_CHANGED_DONE, + "a free key takes the put"); + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_exists_is(txn_entry_tests_checked(&u, "k", txn_entry_tests_put("b", 0), + 1, false, true, simc_begin()), "a"), + "a key holding a committed entry answers its value"); + { + TxnMemtableOp deleted = {.tag = TXN_MEMTABLE_OP_DELETE}; + TxnMemtableVersionWriter writer = txn_entry_tests_writer(&xid); + + txn_entry_tests_changed(io, &made.held, txn_entry_tests_entry(io, &made.storage, writer, 16390, "k", deleted), + __LINE__); + simc_commit(xid); + txn_entry_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + } + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_checked(&u, "k", txn_entry_tests_put("d", 0), 1, false, true, + simc_begin()).tag == TXN_ENTRY_CHANGED_DONE, + "a key whose entry a committed delete ended takes the put again"); + shown = txn_entry_tests_shown(io, &made.held, (const uint8 *) "k", 1); + txn_entry_tests_does_is(__LINE__, &shown, ops, lengthof(ops)); + txn_entry_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_entry_tests_a_counted_unique_entry_writes_nothing_for_its_own_copy_and_answers_another: + * the btree's part 2, slice 5 (orev-14's R2 and R4), a keyless table's + * unique entry counts copies, so a copy under a tuple id the key holds live + * is counted already and writes nothing, and a copy under another tuple id + * answers the live copy's value. + */ +void +test_txn_entry_tests_a_counted_unique_entry_writes_nothing_for_its_own_copy_and_answers_another(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_entry_tests_made_t made = txn_entry_tests_made(io, 9374); + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + txn_entry_tests_unique_t u = txn_entry_tests_unique(io, &made, &order, 16391); + static const char *const ops[] = {"add"}; + StdVec shown; + + for (int k = 0; k < 2; k++) + { + TxnMemtableOp add = txn_entry_tests_put("v", 1); + + add.tag = TXN_MEMTABLE_OP_ADD; + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_checked(&u, "k", add, 1, false, true, simc_begin()).tag == + TXN_ENTRY_CHANGED_DONE, "a copy under a tuple id the key holds live writes nothing"); + } + { + TxnMemtableOp add = txn_entry_tests_put("v", 2); + + add.tag = TXN_MEMTABLE_OP_ADD; + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_exists_is(txn_entry_tests_checked(&u, "k", add, 2, false, true, + simc_begin()), "v"), + "a copy under another tuple id answers the live copy's value"); + } + shown = txn_entry_tests_shown(io, &made.held, (const uint8 *) "k", 1); + txn_entry_tests_does_is(__LINE__, &shown, ops, lengthof(ops)); + txn_entry_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_entry_tests_a_partial_unique_check_answers_a_running_write_without_waiting: + * a speculative insert's partial check (CommPF 10, + * insert-conflict-specconflict) answers another transaction's running write + * of its key as running, writing nothing and waiting for none, as + * _bt_check_unique answers UNIQUE_CHECK_PARTIAL. + */ +void +test_txn_entry_tests_a_partial_unique_check_answers_a_running_write_without_waiting(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_entry_tests_made_t made = txn_entry_tests_made(io, 9375); + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + txn_entry_tests_unique_t u = txn_entry_tests_unique(io, &made, &order, 16392); + static const char *const ops[] = {"put"}; + HostXid first = simc_begin(); + HostXid second = simc_begin(); + StdVec shown; + HostError error; + + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_checked(&u, "k", txn_entry_tests_put("a", 0), 1, false, false, + first).tag == TXN_ENTRY_CHANGED_DONE, + "the first write takes the free key"); + STD_ASSERT_MSG(__LINE__, 9, txn_entry_tests_checked(&u, "k", txn_entry_tests_put("b", 0), 1, true, false, + second).tag == TXN_ENTRY_CHANGED_RUNNING, + "a partial check answers the running write as running"); + simc_commit(first); + simc_commit(second); + txn_entry_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + shown = txn_entry_tests_shown(io, &made.held, (const uint8 *) "k", 1); + txn_entry_tests_does_is(__LINE__, &shown, ops, lengthof(ops)); + txn_entry_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_entry_tests_an_entry_and_its_record_change_together: an index + * that keeps records (CommPF 10's no-user-code removal) changes an entry and + * its record in one change: both pushed under the entry's writer, after + * every entry in the storage's order, and both removed together. + */ +void +test_txn_entry_tests_an_entry_and_its_record_change_together(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_entry_tests_made_t made = txn_entry_tests_made(io, 9376); + static const char *const ops[] = {"delete", "put"}; + StdVec record; + StdVec shown; + HostError error; + + txn_entry_record((const uint8 *) "row", 3, &record); + for (int k = 0; k < 2; k++) + { + HostXid xid; + TxnMemtableVersionWriter writer = txn_entry_tests_writer(&xid); + TxnMemtableOp entry_op = {.tag = TXN_MEMTABLE_OP_DELETE}; + txn_TupleVecU8Op_t recorded = {.f1 = {.tag = TXN_MEMTABLE_OP_DELETE}}; + TxnEntryEntry entry; + + if (k == 0) + { + entry_op = txn_entry_tests_put("v", 0); + recorded.f1 = txn_entry_tests_put("k", 0); + } + entry = txn_entry_tests_entry(io, &made.storage, writer, 16393, "k", entry_op); + std_slice_to_vec_Checked(&recorded.f0, record.ptr, record.len, 1, 1); + txn_entry_tests_ok(txn_entry_change(io, &made.held, &entry, true, &recorded, &error), &error, __LINE__); + simc_commit(xid); + txn_entry_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + } + shown = txn_entry_tests_shown(io, &made.held, (const uint8 *) "k", 1); + txn_entry_tests_does_is(__LINE__, &shown, ops, lengthof(ops)); + shown = txn_entry_tests_shown(io, &made.held, record.ptr, record.len); + txn_entry_tests_does_is(__LINE__, &shown, ops, lengthof(ops)); + STD_ASSERT_MSG(__LINE__, 9, record.len > 0 && ((const uint8 *) record.ptr)[0] == TXN_ENTRY_RECORD && + memcmp(record.ptr, "k", 1) > 0, "record.as_slice() > b\"k\" && record[0] == RECORD"); + std_vec_free_Checked(&record, 1, 1); + txn_entry_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +#endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/fold.c b/src/doltgres/txn/c/fold.c index e1e27dec9dc31..6b32fd775b499 100644 --- a/src/doltgres/txn/c/fold.c +++ b/src/doltgres/txn/c/fold.c @@ -423,7 +423,7 @@ txn_fold_Every_Rule_ended(const void *self, HostIoIo *io, TxnMemtableVersionVers } /* TXN_FOLD_EVERY_RULE_OPS is Every's Rule: every row begun and none ended. */ -static const TxnVisibleRuleOps TXN_FOLD_EVERY_RULE_OPS = { +const TxnVisibleRuleOps TXN_FOLD_EVERY_RULE_OPS = { .begun = txn_fold_Every_Rule_begun, .ended = txn_fold_Every_Rule_ended, }; @@ -888,6 +888,19 @@ txn_fold_ending(HostIoIo *io, TxnVisibleRule rule, TxnMemtableVersionVersion *ch return true; } +/* + * txn_fold_owned returns a pending tuple id's ender word for a row's + * ender's xid: marked TXN_BASE_OWN when it is the row's own writer's, 0 for + * none. + */ +static uint64 +txn_fold_owned(uint64 ender, uint64 writer) +{ + return ender != 0 && ender == writer ? ender | TXN_BASE_OWN : ender; +} + +static void txn_fold_PendingRow_drop(void *row_); + /* TxnFoldFormedTag is Formed's variants, and none before a key's entry is formed. */ typedef enum TxnFoldFormedTag { @@ -948,6 +961,9 @@ typedef struct txn_fold_folded_key_frame_t bool *has_result; TxnFoldKeyFold *result; uint64 *result_ended; + /* gone is a Vec of the delta rows the fold ends. */ + StdVec *gone; + bool gone_live; HostError *error; bool ok; } txn_fold_folded_key_frame_t; @@ -975,7 +991,9 @@ txn_fold_stamp_order(const void *a, const void *b) /* * txn_fold_folded_key_ended reads which rows the rule's appliers ended: each - * delta row it began, for the newest ender, and each base copy, for its + * delta row it began, for the newest ender, and, but in an index's storage, + * made pending by its tuple id with its ender's xid, its heap bytes, and its + * value ids, since it never reaches the base; and each base copy, for its * ender's xid. */ static bool @@ -987,13 +1005,36 @@ txn_fold_folded_key_ended(txn_fold_folded_key_frame_t *frame) { const TxnVisibleSeenRow *seen = &STD_VEC_AT(&frame->every.rows, TxnVisibleSeenRow, i); TxnVisibleDirty dirty = txn_visible_Dirty_Default_default(); + uint64 writer = txn_fold_writer_xid(seen->row.version); bool begun; + TxnFoldPendingRow gone; + TxnMemtableVersionValues values; + uint32 id; if (!frame->rule.ops->begun(frame->rule.self, frame->io, &seen->row, &dirty, &begun, frame->error)) return false; - if (begun && !txn_fold_ending(frame->io, frame->rule, seen->changer, txn_fold_writer_xid(seen->row.version), - &frame->max_ended, &ender, frame->error)) + if (!begun) + continue; + if (!txn_fold_ending(frame->io, frame->rule, seen->changer, writer, &frame->max_ended, &ender, frame->error)) return false; + if (ender == 0 || frame->index) + continue; + gone = (TxnFoldPendingRow) + { + .tuple = seen->row.put.tuple.f0, .ender = txn_fold_owned(ender, writer), + .bytes = seen->row.put.heap_bytes + }; + std_vec_new(&gone.ids, TXN_FOLD_U32); + values = txn_memtable_version_PutRef_values(&seen->row.put); + while (txn_memtable_version_Values_Iterator_next(&values, &id)) + std_vec_push_Checked(&gone.ids, &id, TXN_FOLD_U32); + if (!std_vec_try_reserve(frame->gone, 1, TXN_FOLD_PENDING_ROW)) + { + std_vec_free_Checked(&gone.ids, TXN_FOLD_U32); + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(frame->gone, &gone, TXN_FOLD_PENDING_ROW); } std_vec_new(&frame->enders, TXN_FOLD_U64); frame->enders_live = true; @@ -1046,7 +1087,7 @@ txn_fold_folded_key_kept(txn_fold_folded_key_frame_t *frame, bool *ended) } *ended = true; pended.f0 = stamp->tuple; - pended.f1 = STD_VEC_AT(&frame->enders, uint64, i); + pended.f1 = txn_fold_owned(STD_VEC_AT(&frame->enders, uint64, i), stamp->txn); std_vec_reserve_Checked(&frame->pending, 1, TXN_FOLD_PENDED); std_vec_clone(&pended.f2, &frame->based.ids, TXN_FOLD_U32); std_vec_push_Checked(&frame->pending, &pended, TXN_FOLD_PENDED); @@ -1174,6 +1215,8 @@ txn_fold_folded_key_body(void *frame_) size_t stored_len = 0; fold->has_formed_by = false; + std_vec_new(frame->gone, TXN_FOLD_PENDING_ROW); + frame->gone_live = true; if (!txn_visible_seen(frame->io, frame->rule, frame->versions, frame->versions_len, NULL, &frame->seen, frame->error)) return; @@ -1223,6 +1266,7 @@ txn_fold_folded_key_body(void *frame_) if (!ended && frame->stamps.len == base_copies) { *frame->has_result = false; + frame->gone_live = false; frame->ok = true; return; } @@ -1249,6 +1293,7 @@ txn_fold_folded_key_body(void *frame_) fold->has_formed_by = false; frame->ctids_live = frame->pending_live = false; } + frame->gone_live = false; frame->ok = true; return; } @@ -1271,6 +1316,7 @@ txn_fold_folded_key_body(void *frame_) frame->value_live = frame->addresses_live = frame->stamps_live = frame->formed_ids_live = false; frame->ctids_live = frame->pending_live = false; *frame->has_result = true; + frame->gone_live = false; frame->ok = true; } @@ -1304,6 +1350,9 @@ txn_fold_folded_key_pad(void *frame_) txn_visible_Seen_drop(&frame->every); if (frame->seen_live) txn_visible_Seen_drop(&frame->seen); + if (frame->gone_live) + std_vec_drop_Checked(frame->gone, TXN_FOLD_PENDING_ROW, txn_fold_PendingRow_drop); + frame->gone_live = false; frame->addresses_live = frame->value_live = frame->ctids_live = frame->written_live = false; frame->formed_ids_live = frame->formed_addresses_live = frame->pending_live = frame->stamps_live = false; frame->enders_live = frame->based_live = frame->every_live = frame->seen_live = false; @@ -1321,17 +1370,22 @@ txn_fold_folded_key_pad(void *frame_) * PLAN's form of an index storage's base leaf entry: a keyed table's keeps no * stamp and is one copy of its row, and a keyless table's counts its copies * with a stamp each, by writer and then tuple id; it makes no tuple id edit. + * Last, it answers each delta row the fold applies and an applied version + * ends, which never reaches the base, by its tuple id, with its ender's xid, + * its heap bytes, and its value ids, whose tuple id the fold makes pending, + * so its take leaves the delta with the rest of its block's run (bugs row + * 228's (E)). */ bool txn_fold_folded_key(HostIoIo *io, TreesStoreNodeStore store, bool has_base, TreesStoreTree base, const TreesKeyOrder *order, bool keyless, bool index, TxnVisibleRule rule, const uint8 *key, size_t key_len, TxnMemtableVersionVersion *versions, size_t versions_len, bool *has_result, - TxnFoldKeyFold *result, uint64 *max_ended, HostError *error) + TxnFoldKeyFold *result, uint64 *max_ended, StdVec *gone, HostError *error) { txn_fold_folded_key_frame_t frame = {.io = io, .store = store, .has_base = has_base, .base = base, .order = order, .keyless = keyless, .index = index, .rule = rule, .key = key, .key_len = key_len, .versions = versions, .versions_len = versions_len, .has_result = has_result, .result = result, - .result_ended = max_ended, .error = error}; + .result_ended = max_ended, .gone = gone, .error = error}; std_landing(&frame, txn_fold_folded_key_body, txn_fold_folded_key_pad); txn_fold_folded_key_pad(&frame); @@ -1447,6 +1501,28 @@ txn_fold_PendingRows_insert(StdVec *rows, TxnFoldPendingRow *row) std_vec_insert(rows, low, row, TXN_FOLD_PENDING_ROW, RUST_FILE, 480, 25); } +/* txn_fold_PendingRows_absent inserts a pending tuple id, taking it, only when the rows hold none of its tuple id, dropping it otherwise. */ +static void +txn_fold_PendingRows_absent(StdVec *rows, TxnFoldPendingRow *row) +{ + size_t low = 0; + size_t high = rows->len; + + while (low < high) + { + size_t mid = low + (high - low) / 2; + + if (STD_VEC_AT(rows, TxnFoldPendingRow, mid).tuple < row->tuple) + low = mid + 1; + else + high = mid; + } + if (low < rows->len && STD_VEC_AT(rows, TxnFoldPendingRow, low).tuple == row->tuple) + txn_fold_PendingRow_drop(row); + else + std_vec_insert(rows, low, row, TXN_FOLD_PENDING_ROW, RUST_FILE, 488, 46); +} + /* txn_fold_FoldedRows_drop frees what a fold makes of a storage's rows. */ void txn_fold_FoldedRows_drop(TxnFoldFoldedRows *self) @@ -1619,6 +1695,7 @@ txn_fold_folded_rows_body(void *frame_) bool has_next; bool has_fold; uint64 ended; + StdVec gone; if (!host_delta_hold(frame->io, &frame->hold, frame->error)) return; @@ -1652,9 +1729,12 @@ txn_fold_folded_rows_body(void *frame_) } if (!txn_fold_folded_key(frame->io, frame->store, frame->has_base, frame->base, frame->order, frame->keyless, frame->index, rule, frame->next.f0.ptr, frame->next.f0.len, frame->next.f1.ptr, - frame->next.f1.len, &has_fold, &frame->fold, &ended, frame->error)) + frame->next.f1.len, &has_fold, &frame->fold, &ended, &gone, frame->error)) return; rows->max_ended = Max(rows->max_ended, ended); + for (size_t i = 0; i < gone.len; i++) + txn_fold_PendingRows_absent(&rows->pending, &STD_VEC_AT(&gone, TxnFoldPendingRow, i)); + std_vec_free_Checked(&gone, TXN_FOLD_PENDING_ROW); if (has_fold) { frame->fold_live = true; @@ -1731,6 +1811,7 @@ typedef struct txn_fold_committed_moves_frame_t const TreesKeyOrder *order; bool keyless; bool index; + HostLsn folded; uint64 own; const uint8 *key; size_t key_len; @@ -1755,13 +1836,15 @@ txn_fold_committed_moves_closure_0(txn_fold_committed_moves_frame_t *frame, uint TxnVisibleRule rule = {&TXN_FOLD_COMMITTING_RULE_OPS, &committing}; bool has_fold; uint64 ended; + StdVec gone; memset(added, 0, sizeof(*added)); memset(removed, 0, sizeof(*removed)); if (!txn_fold_folded_key(frame->io, frame->store, frame->has_base, frame->base, frame->order, frame->keyless, frame->index, rule, frame->key, frame->key_len, frame->versions.ptr, - frame->versions.len, &has_fold, &frame->fold, &ended, frame->error)) + frame->versions.len, &has_fold, &frame->fold, &ended, &gone, frame->error)) return false; + std_vec_drop_Checked(&gone, TXN_FOLD_PENDING_ROW, txn_fold_PendingRow_drop); if (!has_fold) return true; frame->fold_live = true; @@ -1803,8 +1886,8 @@ txn_fold_committed_moves_body(void *frame_) if (!host_delta_hold(frame->io, &frame->hold, frame->error)) return; frame->hold_live = true; - if (!txn_memtable_table_versions(frame->io, &frame->hold, host_shared_Held_record(frame->held), frame->key, - frame->key_len, &frame->versions, frame->error)) + if (!txn_visible_versions(frame->io, &frame->hold, host_shared_Held_record(frame->held), frame->key, + frame->key_len, frame->folded, &frame->versions, frame->error)) return; frame->versions_live = true; if (!txn_fold_committed_moves_closure_0(frame, frame->own, &with_added, &with_removed) || @@ -1835,18 +1918,20 @@ txn_fold_committed_moves_pad(void *frame_) * own moves a storage's totals at a key: what the key's fold over the base * writes and gives back with the transaction's versions applied, against * what it does without them, as the sizes it adds and those it takes away, - * neither below zero. + * neither below zero. It folds only the versions a base folded at folded + * didn't apply, so a version a restart's redo replayed under a base that + * holds it counts once (bugs row 323). */ bool txn_fold_committed_moves(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, bool has_base, TreesStoreTree base, bool has_tuple_ids, TreesStoreTree tuple_ids, const TreesKeyOrder *order, - bool keyless, bool index, uint64 own, const uint8 *key, size_t key_len, + bool keyless, bool index, HostLsn folded, uint64 own, const uint8 *key, size_t key_len, TxnBaseSizes *result_0, TxnBaseSizes *result_1, HostError *error) { txn_fold_committed_moves_frame_t frame = {.io = io, .store = store, .held = held, .has_base = has_base, .base = base, .has_tuple_ids = has_tuple_ids, .tuple_ids = tuple_ids, .order = order, .keyless = keyless, - .index = index, .own = own, .key = key, .key_len = key_len, .result_0 = result_0, .result_1 = result_1, - .error = error}; + .index = index, .folded = folded, .own = own, .key = key, .key_len = key_len, .result_0 = result_0, + .result_1 = result_1, .error = error}; std_landing(&frame, txn_fold_committed_moves_body, txn_fold_committed_moves_pad); txn_fold_committed_moves_pad(&frame); @@ -1988,42 +2073,76 @@ txn_fold_committed_value_moves(HostIoIo *io, TreesStoreNodeStore store, const Ho return frame.ok; } -/* txn_TupleHandleWriter_t is ran_key's (Handle, Writer): a version a drop removes, with its writer. */ -typedef struct txn_TupleHandleWriter_t -{ - TxnMemtableVersionHandle f0; - TxnMemtableVersionWriter f1; -} txn_TupleHandleWriter_t; - -/* txn_TupleU32VecTupleHandleWriter_t is Spaced's (u32, Vec<(Handle, Writer)>): a key's versions a drop removes. */ -typedef struct txn_TupleU32VecTupleHandleWriter_t +/* txn_TupleU32VecHandle_t is Spaced's (u32, Vec): a key's versions a drop removes. */ +typedef struct txn_TupleU32VecHandle_t { uint32 f0; StdVec f1; -} txn_TupleU32VecTupleHandleWriter_t; +} txn_TupleU32VecHandle_t; + +/* txn_fold_pending_holds reports whether pending rows, in tuple id order, hold a tuple id. */ +static bool +txn_fold_pending_holds(const StdVec *pending, uint64 tid) +{ + size_t low = 0; + size_t high = pending->len; + + while (low < high) + { + size_t mid = low + (high - low) / 2; + uint64 at = STD_VEC_AT(pending, TxnFoldPendingRow, mid).tuple; + + if (at == tid) + return true; + if (at < tid) + low = mid + 1; + else + high = mid; + } + return false; +} + +/* txn_fold_edit_order orders edits by their keys' bytes, for qsort. */ +static int +txn_fold_edit_order(const void *a, const void *b) +{ + StdBytes x = trees_mutate_Edit_key(a); + StdBytes y = trees_mutate_Edit_key(b); + int c = memcmp(x.ptr, y.ptr, Min(x.len, y.len)); + + if (c != 0) + return c; + return (x.len > y.len) - (x.len < y.len); +} /* * TxnFoldSpaced is Spaced: what the drop through a point takes out of a - * storage's map, the base's tree's edits, each key's versions it removes with - * their writers, whether it removes every version the map holds, and - * whether it met a version whose writer the point leaves for later. + * storage's map, the base's tree's edits, each key's versions it removes, + * whether it removes every version the map holds, and whether it met a + * version whose writer the point leaves for later. */ typedef struct TxnFoldSpaced { /* edits is a Vec. */ StdVec edits; - /* removed is a Vec<(u32, Vec<(Handle, Writer)>)>. */ + /* removed is a Vec<(u32, Vec)>. */ StdVec removed; bool emptied; bool unresolved; + + /* + * kept is a Vec of the spilled versions the drop keeps, each key's + * newest first, which the fold lists in a residual before their files + * leave. + */ + StdVec kept; } TxnFoldSpaced; -#define TXN_FOLD_REMOVED sizeof(txn_TupleU32VecTupleHandleWriter_t), _Alignof(txn_TupleU32VecTupleHandleWriter_t) -#define TXN_FOLD_HANDLE_WRITER sizeof(txn_TupleHandleWriter_t), _Alignof(txn_TupleHandleWriter_t) +#define TXN_FOLD_REMOVED sizeof(txn_TupleU32VecHandle_t), _Alignof(txn_TupleU32VecHandle_t) -/* TxnFoldFits reports whether a run of a key's versions, newest first, may leave the delta. */ -typedef bool (*TxnFoldFits) (void *captures, HostIoIo *io, TxnMemtableVersionVersion *run, size_t run_len, - bool *result, HostError *error); +/* TxnFoldEach reads a run of a key's versions, newest first, that leaves the delta. */ +typedef bool (*TxnFoldEach) (void *captures, HostIoIo *io, TxnMemtableVersionVersion *run, size_t run_len, + HostError *error); /* TxnFoldPaired reports whether the installed base's tuple id tree pairs a tuple id. */ typedef bool (*TxnFoldPaired) (void *captures, HostIoIo *io, uint64 tuple, bool *result, HostError *error); @@ -2032,9 +2151,19 @@ typedef bool (*TxnFoldPaired) (void *captures, HostIoIo *io, uint64 tuple, bool static void txn_fold_Removed_drop(void *removed_) { - txn_TupleU32VecTupleHandleWriter_t *removed = removed_; + txn_TupleU32VecHandle_t *removed = removed_; - std_vec_free_Checked(&removed->f1, TXN_FOLD_HANDLE_WRITER); + std_vec_free_Checked(&removed->f1, TXN_FOLD_HANDLE); +} + +/* txn_fold_Kept_drop frees a kept version's key and bytes. */ +static void +txn_fold_Kept_drop(void *kept_) +{ + TxnSpillKept *kept = kept_; + + std_vec_free_Checked(&kept->key, TXN_FOLD_BYTES); + std_vec_free_Checked(&kept->bytes, TXN_FOLD_BYTES); } /* txn_fold_Spaced_drop frees what a drop takes out of a map. */ @@ -2043,6 +2172,7 @@ txn_fold_Spaced_drop(TxnFoldSpaced *self) { std_vec_drop_Checked(&self->edits, TXN_FOLD_EDIT, trees_mutate_Edit_drop); std_vec_drop_Checked(&self->removed, TXN_FOLD_REMOVED, txn_fold_Removed_drop); + std_vec_drop_Checked(&self->kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept), txn_fold_Kept_drop); } /* txn_fold_ran_key_frame_t is ran_key's state across its pad: the new entry's key, while live. */ @@ -2058,8 +2188,9 @@ typedef struct txn_fold_ran_key_frame_t size_t own; HostLsn point; bool committed; - TxnFoldFits fits; - void *fits_captures; + TxnRunsKind spilled; + TxnFoldEach each; + void *each_captures; StdVec named; bool named_live; HostError *error; @@ -2077,8 +2208,9 @@ txn_fold_ran_key_named(TxnMemtableVersionVersion *version) /* * txn_fold_ran_key_body finds the oldest run of the key's versions the point - * applies or drops, the longest one that fits, and adds its source's entry - * and its memtable versions' removal. + * applies or drops, hands it to each, keeps each spilled version above it + * the point applies or drops, and adds its source's entry and its memtable + * versions' removal. */ static void txn_fold_ran_key_body(void *frame_) @@ -2086,11 +2218,10 @@ txn_fold_ran_key_body(void *frame_) txn_fold_ran_key_frame_t *frame = frame_; TxnFoldSpaced *spaced = frame->spaced; size_t len = frame->versions_len; - size_t longest = 0; - size_t run; + size_t run = 0; size_t first; TxnMemtableVersionVersion *source = NULL; - txn_TupleU32VecTupleHandleWriter_t removed = {.f0 = frame->id}; + txn_TupleU32VecHandle_t removed = {.f0 = frame->id}; for (size_t i = len; i-- > 0;) { @@ -2103,17 +2234,28 @@ txn_fold_ran_key_body(void *frame_) spaced->unresolved = true; break; } - longest++; + run++; } - for (run = longest; run > 0; run--) + if (run > 0 && frame->each != NULL && + !frame->each(frame->each_captures, frame->io, &frame->versions[len - run], run, frame->error)) + return; + for (size_t i = Min(frame->own, len - run); i < len - run; i++) { - bool fits = true; + TxnFoldApplied applied; + TxnSpillKept kept = {.kind = frame->spilled}; - if (frame->fits != NULL && - !frame->fits(frame->fits_captures, frame->io, &frame->versions[len - run], run, &fits, frame->error)) + if (!txn_fold_applied(frame->io, &frame->versions[i], frame->point, &applied, frame->error)) return; - if (fits) - break; + if (applied.tag == TXN_FOLD_APPLIED_STAYS) + continue; + if (!std_vec_try_reserve(&spaced->kept, 1, sizeof(kept), _Alignof(TxnSpillKept))) + { + *frame->error = host_Error_out_of_memory(); + return; + } + txn_fold_bytes(frame->key, frame->key_len, &kept.key); + txn_memtable_version_Version_stored(&frame->versions[i], &kept.bytes); + std_vec_push_Checked(&spaced->kept, &kept, sizeof(kept), _Alignof(TxnSpillKept)); } spaced->emptied &= run == len; if (run == 0) @@ -2175,14 +2317,9 @@ txn_fold_ran_key_body(void *frame_) *frame->error = host_Error_out_of_memory(); return; } - std_vec_new(&removed.f1, TXN_FOLD_HANDLE_WRITER); + std_vec_new(&removed.f1, TXN_FOLD_HANDLE); for (size_t i = first; i < Max(frame->own, first); i++) - { - TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&frame->versions[i]); - txn_TupleHandleWriter_t handle = {frame->versions[i].handle, txn_memtable_version_Record_writer(&record)}; - - std_vec_push_Checked(&removed.f1, &handle, TXN_FOLD_HANDLE_WRITER); - } + std_vec_push_Checked(&removed.f1, &frame->versions[i].handle, TXN_FOLD_HANDLE); std_vec_push_Checked(&spaced->removed, &removed, TXN_FOLD_REMOVED); frame->ok = true; } @@ -2206,11 +2343,11 @@ txn_fold_ran_key_pad(void *frame_) static bool txn_fold_ran_key(HostIoIo *io, TxnFoldSpaced *spaced, const uint8 *key, size_t key_len, uint32 id, TxnMemtableVersionVersion *versions, size_t versions_len, size_t own, HostLsn point, bool committed, - TxnFoldFits fits, void *fits_captures, HostError *error) + TxnRunsKind spilled, TxnFoldEach each, void *each_captures, HostError *error) { txn_fold_ran_key_frame_t frame = {.io = io, .spaced = spaced, .key = key, .key_len = key_len, .id = id, .versions = versions, .versions_len = versions_len, .own = own, .point = point, .committed = committed, - .fits = fits, .fits_captures = fits_captures, .error = error}; + .spilled = spilled, .each = each, .each_captures = each_captures, .error = error}; std_landing(&frame, txn_fold_ran_key_body, txn_fold_ran_key_pad); txn_fold_ran_key_pad(&frame); @@ -2230,8 +2367,8 @@ typedef struct txn_fold_ran_frame_t bool committed; bool has_spilled; TxnRunsKind spilled; - TxnFoldFits fits; - void *fits_captures; + TxnFoldEach each; + void *each_captures; TxnFoldSpaced spaced; bool spaced_live; StdVec walked; @@ -2263,19 +2400,55 @@ typedef struct txn_fold_ran_closure_1_frame_t bool versions_live; StdVec spills; bool spills_live; + /* own is how many of the key's versions the memtable gave. */ + size_t own; HostError *error; bool ok; } txn_fold_ran_closure_1_frame_t; +/* txn_fold_ran_closure_1_dropped drops the key's versions a read gave. */ +static void +txn_fold_ran_closure_1_dropped(void *captures) +{ + txn_fold_ran_closure_1_frame_t *frame = captures; + + frame->versions_live = false; + txn_memtable_version_versions_drop(&frame->versions, 0); +} + +/* txn_fold_ran_closure_1_read reads the key's memtable versions, then its spills' below them. */ +static bool +txn_fold_ran_closure_1_read(void *captures, uint64 *listed, HostError *error) +{ + txn_fold_ran_closure_1_frame_t *frame = captures; + txn_fold_ran_frame_t *ran = frame->ran; + HostSharedRecord record = host_shared_Held_record(ran->held); + + if (!txn_memtable_map_versions(ran->tree, frame->io, &frame->hold, record, std_u32_from_be_bytes(frame->key), + &frame->versions, error)) + return false; + frame->versions_live = true; + frame->own = frame->versions.len; + if (ran->has_spilled) + { + if (!txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, frame->io, &frame->hold, record, ran->spilled, frame->key, + frame->key_len, listed, &frame->spills, error)) + return false; + frame->spills_live = true; + std_vec_extend_from_slice_Checked(&frame->versions, frame->spills.ptr, frame->spills.len, TXN_FOLD_VERSION); + frame->spills_live = false; + std_vec_free_Checked(&frame->spills, TXN_FOLD_VERSION); + } + return true; +} + /* txn_fold_ran_closure_1_body reads the key's memtable versions, then its spills' below them, and adds its run. */ static void txn_fold_ran_closure_1_body(void *frame_) { txn_fold_ran_closure_1_frame_t *frame = frame_; txn_fold_ran_frame_t *ran = frame->ran; - HostSharedRecord record = host_shared_Held_record(ran->held); uint32 id; - size_t own; if (frame->key_len != 4) { @@ -2286,19 +2459,11 @@ txn_fold_ran_closure_1_body(void *frame_) if (!host_delta_hold(frame->io, &frame->hold, frame->error)) return; frame->hold_live = true; - if (!txn_memtable_map_versions(ran->tree, frame->io, &frame->hold, record, id, &frame->versions, frame->error)) + if (!txn_spill_taken(&TXN_SPILL_STATE_HOME_OPS, frame->io, host_shared_Held_record(ran->held), + txn_fold_ran_closure_1_read, txn_fold_ran_closure_1_dropped, frame, frame->error)) return; - frame->versions_live = true; - own = frame->versions.len; if (ran->has_spilled) { - if (!txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, frame->io, &frame->hold, record, ran->spilled, frame->key, - frame->key_len, &frame->spills, frame->error)) - return; - frame->spills_live = true; - std_vec_extend_from_slice_Checked(&frame->versions, frame->spills.ptr, frame->spills.len, TXN_FOLD_VERSION); - frame->spills_live = false; - std_vec_free_Checked(&frame->spills, TXN_FOLD_VERSION); if (!std_vec_try_reserve(&ran->walked, 1, TXN_FOLD_U32)) { *frame->error = host_Error_out_of_memory(); @@ -2307,8 +2472,8 @@ txn_fold_ran_closure_1_body(void *frame_) std_vec_push_Checked(&ran->walked, &id, TXN_FOLD_U32); } frame->ok = txn_fold_ran_key(frame->io, &ran->spaced, frame->key, frame->key_len, id, frame->versions.ptr, - frame->versions.len, own, ran->point, ran->committed, ran->fits, ran->fits_captures, - frame->error); + frame->versions.len, frame->own, ran->point, ran->committed, ran->spilled, ran->each, + ran->each_captures, frame->error); } /* txn_fold_ran_closure_1_pad frees what the walk holds live: the spills' versions, the versions, and its hold. */ @@ -2362,6 +2527,7 @@ txn_fold_ran_body(void *frame_) std_vec_new(&frame->spaced.edits, TXN_FOLD_EDIT); std_vec_new(&frame->spaced.removed, TXN_FOLD_REMOVED); + std_vec_new(&frame->spaced.kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept)); frame->spaced.emptied = true; frame->spaced.unresolved = false; frame->spaced_live = true; @@ -2379,7 +2545,7 @@ txn_fold_ran_body(void *frame_) frame->hold_live = true; if (!txn_spill_next(&TXN_SPILL_STATE_HOME_OPS, frame->io, &frame->hold, record, frame->spilled, frame->has_after, frame->after.ptr, frame->after.len, - TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, &some, &frame->next, frame->error)) + TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, NULL, &some, &frame->next, frame->error)) return; if (!some) break; @@ -2393,7 +2559,7 @@ txn_fold_ran_body(void *frame_) if (bsearch(&id, frame->walked.ptr, frame->walked.len, sizeof(uint32), txn_fold_id_order) == NULL && !txn_fold_ran_key(frame->io, &frame->spaced, frame->next.f0.ptr, frame->next.f0.len, id, frame->next.f1.ptr, frame->next.f1.len, 0, frame->point, frame->committed, - frame->fits, frame->fits_captures, frame->error)) + frame->spilled, frame->each, frame->each_captures, frame->error)) return; if (frame->has_after) std_vec_free_Checked(&frame->after, TXN_FOLD_BYTES); @@ -2404,6 +2570,8 @@ txn_fold_ran_body(void *frame_) frame->hold_live = false; host_delta_Hold_drop(&frame->hold); } + if (frame->has_spilled && frame->spaced.edits.len > 1) + qsort(frame->spaced.edits.ptr, frame->spaced.edits.len, sizeof(TreesMutateEdit), txn_fold_edit_order); *frame->result = frame->spaced; frame->spaced_live = false; frame->ok = true; @@ -2434,20 +2602,22 @@ txn_fold_ran_pad(void *frame_) /* * txn_fold_ran reads what a drop at a fold point takes out of a storage's * map: for each key, the oldest run of versions whose writers the point - * applies or that aborted, the longest such run that fits, whose newest, or - * its newest committed one when committed, none leaving the base's entry, is - * the key's entry in the new base, a removal its deletion, a values mark - * passed over. A map whose spills are of a kind reads each key's spilled - * versions below its memtable's, then the keys its spills alone hold. Only + * applies or that aborted, which it hands each, whose newest, or its newest + * committed one when committed, none leaving the base's entry, is the key's + * entry in the new base, a removal its deletion, a values mark passed over. + * A map whose spills are of a kind reads each key's spilled versions below + * its memtable's, then the keys its spills alone hold, and puts the edits in + * key order, the map's tree's; the spilled versions above a key's run that + * the point applies or drops are kept for a residual (bugs row 228). Only * the memtable's versions are listed for removal. */ static bool txn_fold_ran(HostIoIo *io, const HostSharedHeld *held, const TxnMemtableTableTreeOps *tree, HostLsn point, - bool committed, bool has_spilled, TxnRunsKind spilled, TxnFoldFits fits, void *fits_captures, + bool committed, bool has_spilled, TxnRunsKind spilled, TxnFoldEach each, void *each_captures, TxnFoldSpaced *result, HostError *error) { txn_fold_ran_frame_t frame = {.io = io, .held = held, .tree = tree, .point = point, .committed = committed, - .has_spilled = has_spilled, .spilled = spilled, .fits = fits, .fits_captures = fits_captures, + .has_spilled = has_spilled, .spilled = spilled, .each = each, .each_captures = each_captures, .result = result, .error = error}; std_landing(&frame, txn_fold_ran_body, txn_fold_ran_pad); @@ -2455,99 +2625,92 @@ txn_fold_ran(HostIoIo *io, const HostSharedHeld *held, const TxnMemtableTableTre return frame.ok; } +/* TxnFoldTakenOut is spaced's gathering's captures: the store, the base, the page, the pairing, the fold's pending rows, and the takes no layer frees. */ +typedef struct TxnFoldTakenOut +{ + TreesStoreNodeStore store; + const TxnBaseBase *base; + const TxnInsertPage *page; + TxnFoldPaired paired; + void *paired_captures; + const StdVec *pending; + /* unfreed is a Vec. */ + StdVec *unfreed; +} TxnFoldTakenOut; + /* - * txn_fold_freed reports whether a run of a block's free space versions, - * newest first, may leave the delta: each take in it, and each take's values - * mark, names a tuple id its newest entry holds free, which VACUUM freed, or - * the installed base's tuple id tree pairs. + * txn_fold_taken_out adds to the unfreed each take of a run of a block's + * free space versions, newest first, that leaves the delta naming a tuple id + * no layer frees, as VACUUM's find reads the run's takes (vacuum's leaving): + * neither the run's newest entry holds it free, which VACUUM freed, nor the + * installed base's tuple id tree pairs it, nor the fold makes it pending, + * the fold's ended rows'; so its row aborted, or a prune gave its bytes back + * and left it dead, and only VACUUM frees it, from the base's pending tuple + * ids (bugs row 228's (E)). Each goes with its stored ender, marked + * TXN_BASE_OWN when it is its take's writer's, the bytes its take lowered, 0 + * for a dead one, and its row's value ids when the writer whose commit + * decides them committed. */ static bool -txn_fold_freed(HostIoIo *io, TxnMemtableVersionVersion *run, size_t run_len, HostLsn point, TxnFoldPaired paired, - void *paired_captures, bool *result, HostError *error) +txn_fold_taken_out(void *captures, HostIoIo *io, TxnMemtableVersionVersion *run, size_t run_len, HostError *error) { - bool has_newest = false; - StdBytes newest = {0}; - - for (size_t i = 0; i < run_len; i++) - { - TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&run[i]); - TxnMemtableVersionOpRef op = txn_memtable_version_Record_op(&record); + TxnFoldTakenOut *c = captures; + StdVec leaving; + bool ok = true; - if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_NAMED) - continue; - if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_PUT) - { - has_newest = true; - newest = op.u.put.value; - } - break; - } - *result = true; - for (size_t i = 0; i < run_len; i++) + if (!txn_vacuum_leaving(io, c->store, c->base, c->page, run, run_len, &leaving, error)) + return false; + for (size_t k = 0; k < leaving.len && ok; k++) { - TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&run[i]); - HostRow tuple = txn_memtable_map_taken(txn_memtable_version_Record_op(&record)); - TxnFoldApplied applied; - TxnFoldApplied stays = {.tag = TXN_FOLD_APPLIED_STAYS}; - bool frees = false; + TxnVacuumLeaving *take = &STD_VEC_AT(&leaving, TxnVacuumLeaving, k); + TxnFoldPendingRow row; bool pairs; - if (tuple.f0 == 0) + if (txn_fold_pending_holds(c->pending, take->tid)) continue; - if (!txn_fold_applied(io, &run[i], point, &applied, error)) - return false; - if (txn_fold_Applied_PartialEq_eq(&applied, &stays)) + ok = c->paired(c->paired_captures, io, take->tid, &pairs, error); + if (!ok || pairs) continue; - if (has_newest && - !txn_free_space_frees(newest.ptr, newest.len, (uint16) (tuple.f0 & 0xffff), &frees, error)) - return false; - if (frees) + if (take->bytes > PG_UINT32_MAX) + { + *error = txn_fold_fork("XX001", "a take lowered its entry past a page"); + ok = false; continue; - if (!paired(paired_captures, io, tuple.f0, &pairs, error)) - return false; - if (!pairs) + } + if (!std_vec_try_reserve(c->unfreed, 1, TXN_FOLD_PENDING_ROW)) { - *result = false; - return true; + *error = host_Error_out_of_memory(); + ok = false; + continue; } + row = (TxnFoldPendingRow) + { + .tuple = take->tid, .ender = txn_fold_owned(take->ender, take->writer.xid), .bytes = (uint32) take->bytes, + .ids = take->ids + }; + std_vec_new(&take->ids, TXN_FOLD_U32); + std_vec_push_Checked(c->unfreed, &row, TXN_FOLD_PENDING_ROW); } - return true; -} - -/* txn_fold_spaced_closure_0_captures_t is spaced's fits' captures: the point and the pairing. */ -typedef struct txn_fold_spaced_closure_0_captures_t -{ - HostLsn point; - TxnFoldPaired paired; - void *paired_captures; -} txn_fold_spaced_closure_0_captures_t; - -/* txn_fold_spaced_closure_0 is spaced's fits: whether a run of a block's versions is freed. */ -static bool -txn_fold_spaced_closure_0(void *captures, HostIoIo *io, TxnMemtableVersionVersion *run, size_t run_len, - bool *result, HostError *error) -{ - txn_fold_spaced_closure_0_captures_t *spaced = captures; - - return txn_fold_freed(io, run, run_len, spaced->point, spaced->paired, spaced->paired_captures, result, error); + txn_vacuum_leaving_drop(&leaving); + return ok; } /* * txn_fold_spaced reads what the drop at a fold point takes out of a * storage's free space map: for each block, the oldest run of versions whose - * takers the point applies, each take among them naming a tuple id the run's - * newest entry holds free or the installed base's tuple id tree pairs; the - * newest of the run is the block's entry in the new base, a removal its - * deletion. + * takers the point applies, whose takes each reads before they leave the + * delta, since a take whose tuple id no layer frees goes to the base's + * pending tuple ids (bugs row 228's (E)); the newest of the run is the + * block's entry in the new base, a removal its deletion, so the base stays + * the state before the delta's oldest take of each block. Its spilled + * versions are read below its memtable's. */ static bool -txn_fold_spaced(HostIoIo *io, const HostSharedHeld *held, HostLsn point, TxnFoldPaired paired, - void *paired_captures, TxnFoldSpaced *result, HostError *error) +txn_fold_spaced(HostIoIo *io, const HostSharedHeld *held, HostLsn point, TxnFoldEach each, void *each_captures, + TxnFoldSpaced *result, HostError *error) { - txn_fold_spaced_closure_0_captures_t fits = {point, paired, paired_captures}; - - return txn_fold_ran(io, held, &TXN_MEMTABLE_MAP_FREE_TREE_OPS, point, false, false, 0, - txn_fold_spaced_closure_0, &fits, result, error); + return txn_fold_ran(io, held, &TXN_MEMTABLE_MAP_FREE_TREE_OPS, point, false, true, TXN_RUNS_KIND_SPILLED_FREE, + each, each_captures, result, error); } /* txn_fold_valued_frame_t is valued's state across its pad: what it takes out of the value map, while live. */ @@ -2738,7 +2901,11 @@ typedef struct txn_fold_recount_frame_t bool ok; } txn_fold_recount_frame_t; -/* txn_fold_recount_body folds every committed version over the newest base and moves its sizes by what that writes. */ +/* + * txn_fold_recount_body folds every committed version over the newest base + * and moves its sizes by what that writes; it pauses at + * doltgres-recount-based after it reads the base. + */ static void txn_fold_recount_body(void *frame_) { @@ -2756,6 +2923,8 @@ txn_fold_recount_body(void *frame_) frame->ok = true; return; } + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-recount-based", frame->error)) + return; if (!txn_fold_folded_rows(frame->io, frame->store, frame->held, base->has_root, base->root, base->has_tuple_ids, base->tuple_ids, frame->order, frame->keyless, base->has_index, PG_UINT64_MAX, base->folded, &frame->folded, frame->error)) @@ -2854,7 +3023,7 @@ txn_fold_held_pairing(void *captures, HostIoIo *io, uint64 tuple, bool *result, /* * txn_fold_unspaced_frame_t is unspaced's state across its pad: the keys it - * takes, a key's handles and removal, each while live. + * takes and a key's removal, while live. */ typedef struct txn_fold_unspaced_frame_t { @@ -2863,82 +3032,49 @@ typedef struct txn_fold_unspaced_frame_t const TxnMemtableTableTreeOps *tree; StdVec removed; size_t at; - StdVec handles; - bool handles_live; StdVec taken; bool taken_live; HostError *error; bool ok; } txn_fold_unspaced_frame_t; -/* txn_fold_unspaced_body takes each key's versions out and gives each taken one's count back at its writer's entry. */ +/* txn_fold_unspaced_body takes each key's versions out, the remover giving back each counted one's count. */ static void txn_fold_unspaced_body(void *frame_) { txn_fold_unspaced_frame_t *frame = frame_; - TxnMemtableVersionWriter none = TXN_MEMTABLE_VERSION_NO_TRANSACTION; for (; frame->at < frame->removed.len; frame->at++) { - txn_TupleU32VecTupleHandleWriter_t *key = &STD_VEC_AT(&frame->removed, txn_TupleU32VecTupleHandleWriter_t, - frame->at); + txn_TupleU32VecHandle_t *key = &STD_VEC_AT(&frame->removed, txn_TupleU32VecHandle_t, frame->at); uint8 be[4]; - size_t zipped; - std_vec_with_capacity_Checked(&frame->handles, key->f1.len, TXN_FOLD_HANDLE); - frame->handles_live = true; - for (size_t i = 0; i < key->f1.len; i++) - std_vec_push_Checked(&frame->handles, &STD_VEC_AT(&key->f1, txn_TupleHandleWriter_t, i).f0, - TXN_FOLD_HANDLE); std_u32_to_be_bytes(key->f0, be); if (!txn_memtable_table_removed(frame->tree, frame->io, host_shared_Held_record(frame->held), be, sizeof(be), - frame->handles.ptr, frame->handles.len, &frame->taken, frame->error)) + key->f1.ptr, key->f1.len, &frame->taken, frame->error)) return; - frame->taken_live = true; - zipped = Min(key->f1.len, frame->taken.len); - for (size_t i = 0; i < zipped; i++) - { - const TxnMemtableVersionWriter *writer = &STD_VEC_AT(&key->f1, txn_TupleHandleWriter_t, i).f1; - HostXid top = {writer->has_top ? writer->top : writer->xid}; - bool has_added; - uint64 added; - - if (!STD_VEC_AT(&frame->taken, bool, i) || txn_memtable_version_Writer_PartialEq_eq(writer, &none)) - continue; - if (!txn_status_add(frame->io, top, PG_UINT64_MAX, &has_added, &added, frame->error)) - return; - } - frame->taken_live = false; std_vec_free_Checked(&frame->taken, sizeof(bool), _Alignof(bool)); - frame->handles_live = false; - std_vec_free_Checked(&frame->handles, TXN_FOLD_HANDLE); txn_fold_Removed_drop(key); } frame->ok = true; } -/* txn_fold_unspaced_pad frees what unspaced holds live: a key's removal and handles, and the keys left. */ +/* txn_fold_unspaced_pad frees what unspaced holds live: the keys left. */ static void txn_fold_unspaced_pad(void *frame_) { txn_fold_unspaced_frame_t *frame = frame_; - if (frame->taken_live) - std_vec_free_Checked(&frame->taken, sizeof(bool), _Alignof(bool)); - if (frame->handles_live) - std_vec_free_Checked(&frame->handles, TXN_FOLD_HANDLE); for (; frame->at < frame->removed.len; frame->at++) - txn_fold_Removed_drop(&STD_VEC_AT(&frame->removed, txn_TupleU32VecTupleHandleWriter_t, frame->at)); + txn_fold_Removed_drop(&STD_VEC_AT(&frame->removed, txn_TupleU32VecHandle_t, frame->at)); std_vec_free_Checked(&frame->removed, TXN_FOLD_REMOVED); - frame->taken_live = frame->handles_live = false; } /* * txn_fold_unspaced takes out of a storage's map, through the remover, the * versions an installed base now holds, as spaced or valued read them, - * taking the list, and takes back the count of each version this removal - * took out at its writer's entry; a version written by no transaction counts - * nothing. + * taking the list; the remover gives back the count of each counted version + * it took out. */ static bool txn_fold_unspaced(HostIoIo *io, const HostSharedHeld *held, const TxnMemtableTableTreeOps *tree, StdVec *removed, @@ -3066,6 +3202,7 @@ txn_fold_dropped_closure_0_body(void *frame_) if (seen->changer == NULL || !txn_fold_rows_holds(&frame->unpaired, seen->row.put.tuple)) continue; if (!txn_memtable_map_ended(frame->io, walk->record, seen->row.put.tuple, + txn_fold_writer_xid(seen->row.version), (uint32) txn_fold_writer_xid(seen->changer), frame->error)) return; } @@ -3087,11 +3224,6 @@ txn_fold_dropped_closure_0_body(void *frame_) frame->error)) return; frame->taken_live = true; - for (size_t i = 0; i < Min(frame->done.len, frame->taken.len); i++) - if (STD_VEC_AT(&frame->taken, bool, i) && - !txn_memtable_table_uncounted(frame->io, STD_VEC_AT(&frame->done, TxnMemtableVersionVersion *, i), - frame->error)) - return; frame->ok = true; } @@ -3225,41 +3357,6 @@ txn_fold_pended_closure_0(void *captures, HostIoIo *io, TreesPinPin *pin, Chunks return frame.ok; } -/* txn_fold_pending_holds reports whether pending rows, in tuple id order, hold a tuple id. */ -static bool -txn_fold_pending_holds(const StdVec *pending, uint64 tid) -{ - size_t low = 0; - size_t high = pending->len; - - while (low < high) - { - size_t mid = low + (high - low) / 2; - uint64 at = STD_VEC_AT(pending, TxnFoldPendingRow, mid).tuple; - - if (at == tid) - return true; - if (at < tid) - low = mid + 1; - else - high = mid; - } - return false; -} - -/* txn_fold_edit_order orders edits by their keys' bytes, for qsort. */ -static int -txn_fold_edit_order(const void *a, const void *b) -{ - StdBytes x = trees_mutate_Edit_key(a); - StdBytes y = trees_mutate_Edit_key(b); - int c = memcmp(x.ptr, y.ptr, Min(x.len, y.len)); - - if (c != 0) - return c; - return (x.len > y.len) - (x.len < y.len); -} - /* * txn_fold_pended_frame_t is pended's state across its pad: the tuple ids the * tree holds and the edits, each while live. @@ -3270,8 +3367,9 @@ typedef struct txn_fold_pended_frame_t TreesStoreNodeStore store; const HostSharedHeld *held; const TxnBaseBase *base; - const StdVec *pending; + StdVec *pending; const StdVec *paired; + StdVec *unfreed; StdVec tids; bool tids_live; StdVec edits; @@ -3294,7 +3392,7 @@ static void txn_fold_pended_body(void *frame_) { txn_fold_pended_frame_t *frame = frame_; - size_t left = frame->pending->len; + size_t left; ChunksErrorError failed; std_vec_new(&frame->tids, TXN_FOLD_U64); @@ -3309,6 +3407,17 @@ txn_fold_pended_body(void *frame_) return; } } + for (size_t i = 0; i < frame->unfreed->len; i++) + { + TxnFoldPendingRow *row = &STD_VEC_AT(frame->unfreed, TxnFoldPendingRow, i); + + if (bsearch(&row->tuple, frame->tids.ptr, frame->tids.len, sizeof(uint64), txn_fold_tuple_order) != NULL) + txn_fold_PendingRow_drop(row); + else + txn_fold_PendingRows_absent(frame->pending, row); + } + frame->unfreed->len = 0; + left = frame->pending->len; std_vec_new(&frame->edits, TXN_FOLD_EDIT); frame->edits_live = true; for (size_t i = 0; i < frame->tids.len; i++) @@ -3376,17 +3485,19 @@ txn_fold_pended_pad(void *frame_) /* * txn_fold_pended returns the edits of a storage's pending tuple ids' tree a * fold makes, in key order, and whether the tree holds a tuple id after - * them: a put of each base row's tuple id the fold unpairs, with its ender's - * xid and its heap bytes, and a removal of each pending tuple id the fold - * doesn't unpair that VACUUM has freed or that a row took again, the fold - * pairing it (paired, ascending). + * them: a put of each base row's or delta row's tuple id the fold ends, with + * its ender's xid, its heap bytes, and its row's value ids, and of each + * tuple id no layer frees whose take leaves the delta (unfreed, which it + * takes) that the tree doesn't hold, and a removal of each pending tuple id + * the fold doesn't unpair that VACUUM has freed or that a row took again, + * the fold pairing it (paired, ascending). */ static bool txn_fold_pended(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, const TxnBaseBase *base, - const StdVec *pending, const StdVec *paired, StdVec *result, bool *left, HostError *error) + StdVec *pending, const StdVec *paired, StdVec *unfreed, StdVec *result, bool *left, HostError *error) { txn_fold_pended_frame_t frame = {.io = io, .store = store, .held = held, .base = base, .pending = pending, - .paired = paired, .result = result, .left = left, .error = error}; + .paired = paired, .unfreed = unfreed, .result = result, .left = left, .error = error}; std_landing(&frame, txn_fold_pended_body, txn_fold_pended_pad); txn_fold_pended_pad(&frame); @@ -3424,6 +3535,33 @@ txn_fold_Installed_drop(TxnFoldInstalled *self) self->some = false; } +/* txn_fold_advanced inserts a base's record and gives its end, for an install's logging. */ +static bool +txn_fold_advanced(HostIoIoDoors io, const TxnRecordBaseAdvanced *advanced, HostLsn *end, HostError *error) +{ + StdVec body; + bool ok; + + txn_record_BaseAdvanced_encode(advanced, &body); + ok = host_wal_insert(io, HOST_WAL_KIND_BASE_ADVANCED, body.ptr, body.len, end, error); + std_vec_free_Checked(&body, TXN_FOLD_BYTES); + return ok; +} + +/* + * txn_fold_replayed is a replayed install's logging: it gives the end of the + * record being replayed, which captures names. + */ +static bool +txn_fold_replayed(void *captures, HostIoIoDoors io, HostLsn reached, HostLsn *end, HostError *error) +{ + (void) io; + (void) reached; + (void) error; + *end = *(const HostLsn *) captures; + return true; +} + /* txn_fold_published_drop frees a writer's published roots. */ static void txn_fold_published_drop(StdVec *published) @@ -3437,8 +3575,9 @@ static bool txn_fold_reformed(HostIoIo *io, TreesStoreNodeStore store, const Txn const TxnVisibleLive *chain, size_t chain_len, StdVec *edits, StdVec *formed, StdVec *ctids, TxnFoldReform *reform, StdVec *out_edits, StdVec *out_ctids, TxnBaseSizes *sizes, uint32 *epoch, HostError *error); -static bool txn_fold_matched(HostIoIoDoors io, const HostSharedHeld *held, const TxnBaseBase *folded, - TxnFoldReached *reached, const TreesKeyOrder *order, bool *result, HostError *error); +static bool txn_fold_matched(HostIoIoDoors io, const HostSharedHeld *held, uint64 generation, + const TxnBaseBase *folded, TxnFoldReached *reached, const TreesKeyOrder *order, + bool *result, HostError *error); /* * txn_fold_folded_frame_t is folded's state across its pad: the newest base, @@ -3455,6 +3594,7 @@ typedef struct txn_fold_folded_frame_t const TreesKeyOrder *order; bool keyless; HostLsn point; + const TxnInsertPage *page; TxnFoldReform *reform; TxnFoldReached *reached; uint64 generation; @@ -3503,6 +3643,19 @@ typedef struct txn_fold_folded_frame_t bool paired_now_live; StdVec pends; bool pends_live; + + /* + * unfreed is a Vec of the takes no layer frees that leave the + * delta. + */ + StdVec unfreed; + bool unfreed_live; + /* kept is a Vec of the spilled map versions the drop keeps. */ + StdVec kept; + bool kept_live; + /* residuals is a Vec of the residuals the fold listed. */ + StdVec residuals; + bool residuals_live; bool keeps; bool has_pended; size_t pended; @@ -3571,11 +3724,47 @@ txn_fold_held(TreesStoreBrandedWriter *writer, HostIoTakesShort io, ChunksPacksR return held; } +/* txn_fold_folded_logging_t is a fold's install's logging's captures: the fold, its storage, and its install. */ +typedef struct txn_fold_folded_logging_t +{ + const txn_fold_folded_frame_t *folded; + HostStorage storage; + const TxnFoldInstalled *installed; +} txn_fold_folded_logging_t; + +/* + * txn_fold_folded_logged is a fold's install's logging: outside recovery it + * inserts the installed base's record and gives its end; in recovery it + * gives the later of the point and the end of the record the fold reached. + */ +static bool +txn_fold_folded_logged(void *captures, HostIoIoDoors io, HostLsn reached, HostLsn *end, HostError *error) +{ + const txn_fold_folded_logging_t *logging = captures; + const txn_fold_folded_frame_t *folded = logging->folded; + TxnRecordBaseAdvanced advanced = {.storage = logging->storage, .folded = folded->point, + .max_folded = folded->max_folded, .base = logging->installed->next, + .packs = logging->installed->packs}; + bool injected; + + if (host_wal_stand(io).recovering) + { + *end = Max(reached, folded->point); + return true; + } + if (!host_injected(io, "doltgres-fold-record-root", &injected, error)) + return false; + if (injected && advanced.base.has_root) + advanced.base.root.address = chunks_address_Address_of((const uint8 *) "doltgres-fold-record-root", 25); + return txn_fold_advanced(io, &advanced, end, error); +} + /* * txn_fold_folded_closure_11_body holds the fold's roots in the fold's four * slots, publishes them, and installs the base that names them when the * generation is still the one read before the fold and a standby's roots - * match its primary's. + * match its primary's; a permanent storage's packs are listed first and its + * record logged in the install's step. */ static void txn_fold_folded_closure_11_body(void *frame_) @@ -3590,6 +3779,11 @@ txn_fold_folded_closure_11_body(void *frame_) bool held; bool installed; HostError host; + TxnStatePersistence persistence; + txn_fold_folded_logging_t logging = {.folded = folded, .installed = frame->result}; + bool permanent = txn_state_described(host_shared_Held_record(folded->held), &logging.storage, + &persistence) && + persistence == TXN_STATE_PERSISTENCE_PERMANENT; chunks_packs_roots_RootSlots_fold(frame->slots, &slot); held = txn_fold_held(&folded->writer, io, slot.s0, true, folded->root, true, frame->error); @@ -3623,9 +3817,15 @@ txn_fold_folded_closure_11_body(void *frame_) next->pending = trees_published_Published_base(&STD_VEC_AT(&frame->published, TreesPublishedPublished, at++)); } - next->counter = folded->counter; - next->fresh_end = folded->fresh_end; + + /* + * A standby's replay may already hold versions past the point, so its + * counter and fresh end are its primary's at the point, from the record. + */ + next->counter = folded->reached != NULL ? folded->reached->base->counter : folded->counter; + next->fresh_end = folded->reached != NULL ? folded->reached->base->fresh_end : folded->fresh_end; next->folded = folded->point; + next->read = folded->point; next->max_folded = folded->max_folded; next->max_ended = Max(folded->base.max_ended, folded->rows.max_ended); next->sizes = folded->sizes; @@ -3635,7 +3835,8 @@ txn_fold_folded_closure_11_body(void *frame_) { bool matched; - if (!txn_fold_matched(doors, folded->held, next, folded->reached, folded->order, &matched, &host)) + if (!txn_fold_matched(doors, folded->held, folded->generation, next, folded->reached, folded->order, + &matched, &host)) { *frame->error = chunks_error_Error_FromError_from(&host); return; @@ -3647,7 +3848,11 @@ txn_fold_folded_closure_11_body(void *frame_) return; } } - if (!txn_state_installed(io, folded->held, folded->generation, next, &installed, &host)) + if (permanent && + !chunks_packs_backend_Packs_flush(&folded->repo->packs, io, folded->reading, frame->error)) + return; + if (!txn_state_installed(io, folded->held, folded->generation, next, permanent ? txn_fold_folded_logged : NULL, + &logging, &installed, &host)) { *frame->error = chunks_error_Error_FromError_from(&host); return; @@ -3871,9 +4076,16 @@ txn_fold_folded_write(txn_fold_folded_frame_t *frame) frame->held_pairing.store = store; frame->held_pairing.has_tuple_ids = frame->base.has_tuple_ids; frame->held_pairing.tuple_ids = frame->base.tuple_ids; - if (!txn_fold_spaced(frame->io, frame->held, frame->point, txn_fold_folded_closure_8, frame, &frame->spaces, - frame->error)) - return false; + std_vec_new(&frame->unfreed, TXN_FOLD_PENDING_ROW); + frame->unfreed_live = true; + { + TxnFoldTakenOut gathered = {store, &frame->base, frame->page, txn_fold_folded_closure_8, frame, + &frame->rows.pending, &frame->unfreed}; + + if (!txn_fold_spaced(frame->io, frame->held, frame->point, txn_fold_taken_out, &gathered, &frame->spaces, + frame->error)) + return false; + } frame->spaces_live = true; frame->has_spaced = frame->spaces.edits.len > 0; if (frame->has_spaced) @@ -3905,7 +4117,7 @@ txn_fold_folded_write(txn_fold_folded_frame_t *frame) std_vec_push_Checked(&frame->paired_now, &STD_VEC_AT(&frame->pairs, txn_TupleU64Bool_t, i).f0, TXN_FOLD_U64); if (!txn_fold_pended(frame->io, store, frame->held, &frame->base, &frame->rows.pending, &frame->paired_now, - &frame->pends, &left, frame->error)) + &frame->unfreed, &frame->pends, &left, frame->error)) return false; frame->pends_live = true; frame->keeps = frame->pends.len == 0; @@ -3924,63 +4136,25 @@ txn_fold_folded_write(txn_fold_folded_frame_t *frame) return true; } -/* - * txn_fold_folded_logged flushes a permanent storage's packs and the WAL - * through the point, and outside recovery logs the installed base's record, - * taking the install. - */ +/* txn_fold_folded_flushed flushes the WAL through the point for a permanent storage. */ static bool -txn_fold_folded_logged(txn_fold_folded_frame_t *frame) +txn_fold_folded_flushed(txn_fold_folded_frame_t *frame) { - HostIoIoDoors doors = HOST_IO_IO_IO_DOORS(frame->io); HostStorage storage; TxnStatePersistence persistence; - ChunksErrorError failed; - TxnRecordBaseAdvanced advanced; - bool injected; - StdVec body; - HostLsn lsn; - bool ok; if (!txn_state_described(host_shared_Held_record(frame->held), &storage, &persistence) || persistence != TXN_STATE_PERSISTENCE_PERMANENT) return true; - if (!chunks_packs_backend_Packs_flush(&frame->repo->packs, frame->io, frame->reading, &failed)) - { - *frame->error = txn_base_storage(&failed); - return false; - } - if (!host_wal_flush(doors, frame->point, frame->error)) - return false; - if (host_wal_stand(doors).recovering) - return true; - advanced.storage = storage; - advanced.folded = frame->point; - advanced.max_folded = frame->max_folded; - advanced.base = frame->installed.next; - advanced.packs = frame->installed.packs; - advanced.built = false; - advanced.horizon = 0; - advanced.has_chain = false; - frame->installed.some = false; - ok = host_injected(doors, "doltgres-fold-record-root", &injected, frame->error); - if (ok && injected && advanced.base.has_root) - advanced.base.root.address = chunks_address_Address_of((const uint8 *) "doltgres-fold-record-root", 25); - if (ok) - { - txn_record_BaseAdvanced_encode(&advanced, &body); - ok = host_wal_insert(doors, HOST_WAL_KIND_BASE_ADVANCED, body.ptr, body.len, &lsn, frame->error); - std_vec_free_Checked(&body, TXN_FOLD_BYTES); - } - txn_record_BaseAdvanced_drop(&advanced); - return ok; + return host_wal_flush(HOST_IO_IO_IO_DOORS(frame->io), frame->point, frame->error); } /* * txn_fold_folded_body folds the storage's rows and maps at the point, * re-forming its rows through a live chain, writes the new base's trees, and - * installs the base under the registration that holds its roots, then gives - * back the chain's counts, logs the base, and drops what the install folded. + * installs and logs the base under the registration that holds its roots, + * then gives back the chain's counts, flushes the WAL through the point, and + * drops what the install folded. */ static void txn_fold_folded_body(void *frame_) @@ -4004,7 +4178,7 @@ txn_fold_folded_body(void *frame_) store = TREES_STORE_READER_NODE_STORE(&frame->reader); if (!txn_fold_folded_rows(frame->io, store, frame->held, frame->base.has_root, frame->base.root, frame->base.has_tuple_ids, frame->base.tuple_ids, frame->order, frame->keyless, - frame->base.has_index, frame->point, frame->base.folded, &frame->rows, frame->error)) + frame->base.has_index, frame->point, frame->base.read, &frame->rows, frame->error)) return; frame->rows_live = true; if (frame->rows.overlapping) @@ -4043,7 +4217,7 @@ txn_fold_folded_body(void *frame_) { txn_fold_held_pairing_t pairing = {store, frame->base.has_tuple_ids, frame->base.tuple_ids}; - frame->ok = txn_fold_dropped(frame->io, frame->held, frame->base.folded, txn_fold_held_pairing, &pairing, + frame->ok = txn_fold_dropped(frame->io, frame->held, frame->base.read, txn_fold_held_pairing, &pairing, frame->error) && txn_memtable_tuples_emptied(frame->io, record, frame->error); return; @@ -4069,6 +4243,26 @@ txn_fold_folded_body(void *frame_) std_vec_new(&frame->values.edits, TXN_FOLD_EDIT); if (!txn_fold_folded_sort(frame) || !txn_fold_folded_write(frame)) return; + std_vec_new(&frame->kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept)); + frame->kept_live = true; + if (!std_vec_try_reserve_exact(&frame->kept, frame->spaces.kept.len + frame->values.kept.len, + sizeof(TxnSpillKept), _Alignof(TxnSpillKept))) + { + *frame->error = host_Error_out_of_memory(); + return; + } + std_vec_extend_from_slice_Checked(&frame->kept, frame->spaces.kept.ptr, frame->spaces.kept.len, + sizeof(TxnSpillKept), _Alignof(TxnSpillKept)); + frame->spaces.kept.len = 0; + std_vec_extend_from_slice_Checked(&frame->kept, frame->values.kept.ptr, frame->values.kept.len, + sizeof(TxnSpillKept), _Alignof(TxnSpillKept)); + frame->values.kept.len = 0; + if (!txn_spill_residual(frame->io, record, frame->point, frame->kept.ptr, frame->kept.len, &frame->residuals, + frame->error)) + return; + frame->residuals_live = true; + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-fold-residual-listed", frame->error)) + return; if (!chunks_packs_roots_RootSlots_register(frame->io, txn_fold_folded_closure_11, frame, NULL, &frame->installed, &failed)) { @@ -4077,7 +4271,7 @@ txn_fold_folded_body(void *frame_) } if (!frame->installed.some) { - frame->ok = true; + frame->ok = txn_spill_unlisted(frame->io, record, &frame->residuals, frame->error); return; } for (size_t i = 0; i < frame->chained.len; i++) @@ -4089,18 +4283,20 @@ txn_fold_folded_body(void *frame_) if (!txn_status_add(frame->io, xid, PG_UINT64_MAX, &has_added, &added, frame->error)) return; } - if (!txn_fold_folded_logged(frame) || - !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-fold-installed", frame->error) || + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-fold-installed", frame->error) || + !txn_fold_folded_flushed(frame) || !txn_fold_dropped(frame->io, frame->held, frame->point, txn_fold_folded_closure_8, frame, frame->error) || !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-drop-installed", frame->error)) return; frame->spaces_live = false; std_vec_free_Checked(&frame->spaces.edits, TXN_FOLD_EDIT); + std_vec_free_Checked(&frame->spaces.kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept)); if (!txn_fold_unspaced(frame->io, frame->held, &TXN_MEMTABLE_MAP_FREE_TREE_OPS, &frame->spaces.removed, frame->error)) return; frame->values_live = false; std_vec_free_Checked(&frame->values.edits, TXN_FOLD_EDIT); + std_vec_free_Checked(&frame->values.kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept)); if (!txn_fold_unspaced(frame->io, frame->held, &TXN_MEMTABLE_MAP_VALUES_TREE_OPS, &frame->values.removed, frame->error) || !txn_memtable_tuples_emptied(frame->io, record, frame->error)) @@ -4116,6 +4312,13 @@ txn_fold_folded_pad(void *frame_) txn_fold_folded_frame_t *frame = frame_; txn_fold_Installed_drop(&frame->installed); + if (frame->residuals_live) + std_vec_free_Checked(&frame->residuals, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (frame->kept_live) + std_vec_drop_Checked(&frame->kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept), txn_fold_Kept_drop); + if (frame->unfreed_live) + std_vec_drop_Checked(&frame->unfreed, TXN_FOLD_PENDING_ROW, txn_fold_PendingRow_drop); + frame->residuals_live = frame->kept_live = frame->unfreed_live = false; if (frame->pends_live) std_vec_drop_Checked(&frame->pends, TXN_FOLD_EDIT, trees_mutate_Edit_drop); if (frame->paired_now_live) @@ -4171,16 +4374,18 @@ txn_fold_folded_pad(void *frame_) * with more than one row refuses its install. A base whose chain holds a live * transform installs only with a re-form and every live transform's commit * at or below the point. A fold in recovery logs no record, and with reached - * installs only a base whose roots matched accepts. It answers whether it + * installs only a base whose roots matched accepts. The spilled map + * versions its drop keeps go to residuals listed before it installs, taken + * back out when its install is refused (bugs row 228). It answers whether it * installed. */ bool txn_fold_folded(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, - const TreesKeyOrder *order, bool keyless, HostLsn point, TxnFoldReform *reform, - TxnFoldReached *reached, bool *result, HostError *error) + const TreesKeyOrder *order, bool keyless, HostLsn point, const TxnInsertPage *page, + TxnFoldReform *reform, TxnFoldReached *reached, bool *result, HostError *error) { txn_fold_folded_frame_t frame = {.io = io, .repo = repo, .reading = reading, .held = held, .order = order, - .keyless = keyless, .point = point, .reform = reform, .reached = reached, .result = result, + .keyless = keyless, .point = point, .page = page, .reform = reform, .reached = reached, .result = result, .error = error}; std_landing(&frame, txn_fold_folded_body, txn_fold_folded_pad); @@ -4234,7 +4439,7 @@ txn_fold_fold_now(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *re if (!has_point || left) return true; if (!txn_fold_bounded(io, held, point, horizon, &bounded, error) || - !txn_fold_folded(io, repo, reading, held, order, keyless, bounded, reform, NULL, &folded, error)) + !txn_fold_folded(io, repo, reading, held, order, keyless, bounded, page, reform, NULL, &folded, error)) return false; *result = folded || installed; return true; @@ -4280,7 +4485,7 @@ txn_fold_in_order_body(void *frame_) bool folded; if (!txn_fold_folded(frame->io, frame->repo, frame->reading, frame->held, frame->order, frame->keyless, - frame->base.folded, NULL, &reached, &folded, frame->error)) + frame->base.folded, frame->page, NULL, &reached, &folded, frame->error)) return; frame->base_live = false; txn_base_Base_drop(&frame->base); @@ -4574,11 +4779,15 @@ txn_fold_shown(bool has_root, const ChunksAddressAddress *root, StdString *resul * installs it. A byte-ordered tree whose root differs publishes the * mismatch, whose FATAL ends recovery, and the fold installs nothing; a rows * tree in a comparator's order whose root differs is reported and the - * standby keeps its own. + * standby keeps its own. A fold whose state's base generation moved past the + * one it read its base at compares nothing and installs nothing, as its + * install would fail, since a newer install's drop may have taken what it + * read (bugs row 249); with the generation unmoved, nothing it read has + * changed since, so a mismatch it finds is its primary's. */ static bool -txn_fold_matched(HostIoIoDoors io, const HostSharedHeld *held, const TxnBaseBase *folded, TxnFoldReached *reached, - const TreesKeyOrder *order, bool *result, HostError *error) +txn_fold_matched(HostIoIoDoors io, const HostSharedHeld *held, uint64 generation, const TxnBaseBase *folded, + TxnFoldReached *reached, const TreesKeyOrder *order, bool *result, HostError *error) { static const char *const names[] = {"rows", "tuple id", "free space", "pending tuple id"}; const TxnBaseBase *theirs = reached->base; @@ -4591,8 +4800,8 @@ txn_fold_matched(HostIoIoDoors io, const HostSharedHeld *held, const TxnBaseBase HostStorage storage; TxnStatePersistence persistence; - *result = true; - if (!txn_state_described(host_shared_Held_record(held), &storage, &persistence)) + *result = txn_state_generation(host_shared_Held_record(held)) == generation; + if (!*result || !txn_state_described(host_shared_Held_record(held), &storage, &persistence)) return true; for (int at = 0; at < lengthof(names); at++) { @@ -4800,8 +5009,9 @@ txn_fold_bounded_map(HostIoIo *io, HostSharedRecord record, const TxnMemtableTab * txn_fold_bounded lowers a fold point so the fold applies no top-level * transaction at or past a horizon: below the commit end, at or below the * point, of each committed writer at or past the horizon among the storage's - * row versions its newest base didn't apply and its free space and value - * maps' versions, every version a fold applies. + * row versions its newest base didn't apply, its free space and value maps' + * versions, and its listed spills' transactions, every version a fold + * applies. */ static bool txn_fold_bounded(HostIoIo *io, const HostSharedHeld *held, HostLsn point, uint64 horizon, HostLsn *result, @@ -4811,19 +5021,32 @@ txn_fold_bounded(HostIoIo *io, const HostSharedHeld *held, HostLsn point, uint64 txn_fold_bounded_captures_t walk = {.record = record, .horizon = horizon, .bound = result}; bool some; TxnBaseBase newest; + HostStorage storage; + TxnStatePersistence persistence; + txn_TupleVecFileU64_t listed; + bool ok = true; *result = point; if (!txn_state_newest(io, held, &some, &newest, error)) return false; if (some) { - walk.folded = newest.folded; + walk.folded = newest.read; txn_base_Base_drop(&newest); } - return txn_memtable_table_walked(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, record, txn_fold_bounded_closure_1, &walk, - NULL, error) && - txn_fold_bounded_map(io, record, &TXN_MEMTABLE_MAP_FREE_TREE_OPS, horizon, result, error) && - txn_fold_bounded_map(io, record, &TXN_MEMTABLE_MAP_VALUES_TREE_OPS, horizon, result, error); + if (!txn_memtable_table_walked(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, record, txn_fold_bounded_closure_1, &walk, + NULL, error) || + !txn_fold_bounded_map(io, record, &TXN_MEMTABLE_MAP_FREE_TREE_OPS, horizon, result, error) || + !txn_fold_bounded_map(io, record, &TXN_MEMTABLE_MAP_VALUES_TREE_OPS, horizon, result, error)) + return false; + if (!txn_state_described(record, &storage, &persistence)) + return true; + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, record, &storage, &listed, error)) + return false; + for (size_t k = 0; k < listed.f0.len && ok; k++) + ok = txn_fold_kept_out(io, STD_VEC_AT(&listed.f0, TxnSpillFile, k).xid, horizon, result, error); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + return ok; } /* @@ -4842,6 +5065,8 @@ typedef struct txn_fold_rechained_base_frame_t bool base_live; const TxnBasePending *entries; size_t entries_len; + TxnStateLogged logged; + void *logged_captures; TreesStoreWriter plain; TreesStoreBrandedWriter writer; bool writer_live; @@ -4870,7 +5095,11 @@ typedef struct txn_fold_rechained_base_closure_1_frame_t bool ok; } txn_fold_rechained_base_closure_1_frame_t; -/* txn_fold_rechained_base_closure_1_body holds the chain's root in one slot, publishes it, and replaces the base. */ +/* + * txn_fold_rechained_base_closure_1_body holds the chain's root in one slot, + * publishes it, and replaces the base, listing its pack first when the + * replace logs. + */ static void txn_fold_rechained_base_closure_1_body(void *frame_) { @@ -4895,7 +5124,11 @@ txn_fold_rechained_base_closure_1_body(void *frame_) frame->result->next.has_chain = true; frame->result->next.chain = trees_published_Published_base(&STD_VEC_AT(&frame->published, TreesPublishedPublished, 0)); - if (!txn_state_rechained(io, rechained->held, rechained->generation, &frame->result->next, &replaced, &host)) + if (rechained->logged != NULL && + !chunks_packs_backend_Packs_flush(&rechained->repo->packs, io, rechained->reading, frame->error)) + return; + if (!txn_state_rechained(io, rechained->held, rechained->generation, &frame->result->next, rechained->logged, + rechained->logged_captures, &replaced, &host)) { *frame->error = chunks_error_Error_FromError_from(&host); return; @@ -5027,16 +5260,17 @@ txn_fold_rechained_base_pad(void *frame_) * replaces a state's newest base, which it takes, with one naming it when the * base generation is still the one read with the base, answering the new * base and the packs it wrote, or nothing when another install moved the - * generation. + * generation. The replace logs through logged, when given. */ static bool txn_fold_rechained_base(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, uint64 generation, TxnBaseBase *base, - const TxnBasePending *entries, size_t entries_len, TxnFoldInstalled *result, HostError *error) + const TxnBasePending *entries, size_t entries_len, TxnStateLogged logged, + void *logged_captures, TxnFoldInstalled *result, HostError *error) { txn_fold_rechained_base_frame_t frame = {.io = io, .repo = repo, .reading = reading, .held = held, .generation = generation, .base = *base, .base_live = true, .entries = entries, .entries_len = entries_len, - .result = result, .error = error}; + .logged = logged, .logged_captures = logged_captures, .result = result, .error = error}; result->some = false; std_landing(&frame, txn_fold_rechained_base_body, txn_fold_rechained_base_pad); @@ -5061,6 +5295,10 @@ typedef struct txn_fold_chain_frame_t size_t recorded_entries_len; TxnBaseBase recorded; bool recorded_live; + /* end is a replayed chain record's end. */ + HostLsn end; + /* storage is a DDL's storage, which its record names. */ + HostStorage storage; TxnBaseBase base; bool base_live; StdVec current; @@ -5077,6 +5315,22 @@ typedef struct txn_fold_chain_frame_t bool ok; } txn_fold_chain_frame_t; +/* + * txn_fold_chain_logged is a DDL's replace's logging: it inserts the record + * of the pass's base and its chain's entries, and gives its end. + */ +static bool +txn_fold_chain_logged(void *captures, HostIoIoDoors io, HostLsn reached, HostLsn *end, HostError *error) +{ + const txn_fold_chain_frame_t *frame = captures; + TxnRecordBaseAdvanced advanced = {.storage = frame->storage, .folded = frame->installed.next.folded, + .max_folded = frame->installed.next.max_folded, .base = frame->installed.next, + .packs = frame->installed.packs, .has_chain = true, .chain = frame->entries}; + + (void) reached; + return txn_fold_advanced(io, &advanced, end, error); +} + /* txn_fold_chain_pass_drop frees what a pass of a chain loop holds: its install, counts, entries, chain, and base. */ static void txn_fold_chain_pass_drop(txn_fold_chain_frame_t *frame) @@ -5177,8 +5431,8 @@ txn_fold_chained_body(void *frame_) HostXid xid = {frame->pending.xid}; bool has_added; uint64 added; - HostStorage storage; TxnStatePersistence persistence; + bool permanent; txn_fold_chain_pass_drop(frame); if (!txn_fold_chain_read(frame, "a DDL's transform found its storage with no base", &some, &generation)) @@ -5211,34 +5465,17 @@ txn_fold_chained_body(void *frame_) txn_base_Pending_Clone_clone(&frame->pending, &pending); std_vec_push_Checked(&frame->entries, &pending, TXN_FOLD_CHAIN); frame->base_live = false; + permanent = txn_state_described(host_shared_Held_record(frame->held), &frame->storage, &persistence) && + persistence == TXN_STATE_PERSISTENCE_PERMANENT; if (!txn_fold_rechained_base(frame->io, frame->repo, frame->reading, frame->held, generation, &frame->base, - frame->entries.ptr, frame->entries.len, &frame->installed, frame->error)) + frame->entries.ptr, frame->entries.len, permanent ? txn_fold_chain_logged : NULL, + frame, &frame->installed, frame->error)) return; if (!frame->installed.some) continue; if (!txn_fold_chain_given(frame) || !txn_status_add(frame->io, xid, 1, &has_added, &added, frame->error)) return; - if (txn_state_described(host_shared_Held_record(frame->held), &storage, &persistence) && - persistence == TXN_STATE_PERSISTENCE_PERMANENT) - { - TxnRecordBaseAdvanced advanced = {.storage = storage, .folded = frame->installed.next.folded, - .max_folded = frame->installed.next.max_folded, .base = frame->installed.next, - .packs = frame->installed.packs, .has_chain = true, .chain = frame->entries}; - StdVec body; - HostLsn lsn; - bool ok; - - frame->installed.some = false; - frame->entries_live = false; - txn_record_BaseAdvanced_encode(&advanced, &body); - ok = host_wal_insert(HOST_IO_IO_IO_DOORS(frame->io), HOST_WAL_KIND_BASE_ADVANCED, body.ptr, body.len, - &lsn, frame->error); - std_vec_free_Checked(&body, TXN_FOLD_BYTES); - txn_record_BaseAdvanced_drop(&advanced); - if (!ok) - return; - } frame->ok = true; return; } @@ -5314,7 +5551,8 @@ txn_fold_redo_chained_body(void *frame_) } frame->base_live = false; if (!txn_fold_rechained_base(frame->io, frame->repo, frame->reading, frame->held, generation, &frame->base, - frame->entries.ptr, frame->entries.len, &frame->installed, frame->error)) + frame->entries.ptr, frame->entries.len, txn_fold_replayed, &frame->end, + &frame->installed, frame->error)) return; if (!frame->installed.some) continue; @@ -5360,15 +5598,16 @@ txn_fold_redo_chained_body(void *frame_) * server's save wrote; each entry without an end the replaced chain lacked * counts at its top-level entry, and each without one it held that the * record lacks gives its count back. A storage this server holds no base of - * yet takes the record's base, which it takes, with no chain. + * yet takes the record's base, which it takes, with no chain. The replaced + * base keeps the record's end. */ bool txn_fold_redo_chained(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, const TxnBasePending *entries, size_t entries_len, - TxnBaseBase *recorded, HostError *error) + TxnBaseBase *recorded, HostLsn end, HostError *error) { txn_fold_chain_frame_t frame = {.io = io, .repo = repo, .reading = reading, .held = held, - .recorded_entries = entries, .recorded_entries_len = entries_len, .recorded = *recorded, + .recorded_entries = entries, .recorded_entries_len = entries_len, .recorded = *recorded, .end = end, .recorded_live = true, .error = error}; std_landing(&frame, txn_fold_redo_chained_body, txn_fold_chain_pad); @@ -5455,7 +5694,8 @@ txn_fold_settled_body(void *frame_) break; frame->base_live = false; if (!txn_fold_rechained_base(frame->io, frame->repo, frame->reading, frame->held, generation, &frame->base, - frame->entries.ptr, frame->entries.len, &frame->installed, frame->error)) + frame->entries.ptr, frame->entries.len, NULL, NULL, &frame->installed, + frame->error)) return; if (!frame->installed.some) continue; @@ -5513,6 +5753,8 @@ typedef struct txn_fold_whole_frame_t TxnBaseBase base; bool base_live; HostLsn point; + /* read is the installed base's read point. */ + HostLsn read; uint64 generation; uint64 horizon; bool has_nodes; @@ -5527,6 +5769,30 @@ typedef struct txn_fold_whole_frame_t bool ok; } txn_fold_whole_frame_t; +/* txn_fold_whole_logging_t is a built base's install's logging's captures: the build, its storage, and its install. */ +typedef struct txn_fold_whole_logging_t +{ + const txn_fold_whole_frame_t *whole; + HostStorage storage; + const TxnFoldInstalled *installed; +} txn_fold_whole_logging_t; + +/* + * txn_fold_whole_advanced is a built base's install's logging: it inserts + * the base's record, after its nodes', and gives its end. + */ +static bool +txn_fold_whole_advanced(void *captures, HostIoIoDoors io, HostLsn reached, HostLsn *end, HostError *error) +{ + const txn_fold_whole_logging_t *logging = captures; + TxnRecordBaseAdvanced advanced = {.storage = logging->storage, .folded = logging->whole->point, + .max_folded = logging->installed->next.max_folded, .base = logging->installed->next, + .packs = logging->installed->packs, .built = true, .horizon = logging->whole->horizon}; + + (void) reached; + return txn_fold_advanced(io, &advanced, end, error); +} + /* txn_fold_whole_closure_0_frame_t is whole's run's state across its pad: the roots it published. */ typedef struct txn_fold_whole_closure_0_frame_t { @@ -5559,6 +5825,11 @@ txn_fold_whole_closure_0_body(void *frame_) HostError host; bool *has[4]; TreesStoreTree *named[4]; + TxnStatePersistence persistence; + txn_fold_whole_logging_t logging = {.whole = whole, .installed = frame->result}; + bool permanent = txn_state_described(host_shared_Held_record(whole->held), &logging.storage, + &persistence) && + persistence == TXN_STATE_PERSISTENCE_PERMANENT; chunks_packs_roots_RootSlots_fold(frame->slots, &slot); held = txn_fold_held(whole->writer, io, slot.s0, whole->has_roots[0], whole->roots[0], true, frame->error); @@ -5590,7 +5861,11 @@ txn_fold_whole_closure_0_body(void *frame_) *named[i] = trees_published_Published_base(&STD_VEC_AT(&frame->published, TreesPublishedPublished, at++)); } next->folded = whole->point; - if (!txn_state_installed(io, whole->held, whole->generation, next, &installed, &host)) + next->read = whole->read; + if (permanent && !chunks_packs_backend_Packs_flush(&whole->repo->packs, io, whole->reading, frame->error)) + return; + if (!txn_state_installed(io, whole->held, whole->generation, next, permanent ? txn_fold_whole_advanced : NULL, + &logging, &installed, &host)) { *frame->error = chunks_error_Error_FromError_from(&host); return; @@ -5661,9 +5936,9 @@ txn_fold_Logged_drop(void *logged_) } /* - * txn_fold_whole_logged logs a permanent storage's built base: its packs and - * the WAL flushed, each node the writer kept as a Chunk record in its own - * checkpoint window, children before parents, then the base's record. + * txn_fold_whole_logged logs a permanent storage's built base's nodes: the + * WAL flushed, each node the writer kept as a Chunk record in its own + * checkpoint window, children before parents. */ static bool txn_fold_whole_logged(txn_fold_whole_frame_t *frame) @@ -5671,8 +5946,6 @@ txn_fold_whole_logged(txn_fold_whole_frame_t *frame) HostIoIoDoors doors = HOST_IO_IO_IO_DOORS(frame->io); HostStorage storage; TxnStatePersistence persistence; - ChunksErrorError failed; - TxnRecordBaseAdvanced advanced; StdVec body; HostLsn lsn; bool ok; @@ -5680,11 +5953,6 @@ txn_fold_whole_logged(txn_fold_whole_frame_t *frame) if (!txn_state_described(host_shared_Held_record(frame->held), &storage, &persistence) || persistence != TXN_STATE_PERSISTENCE_PERMANENT) return true; - if (!chunks_packs_backend_Packs_flush(&frame->repo->packs, frame->io, frame->reading, &failed)) - { - *frame->error = txn_base_storage(&failed); - return false; - } if (!host_wal_flush(doors, frame->point, frame->error)) return false; if (storage.database == 0) @@ -5723,25 +5991,13 @@ txn_fold_whole_logged(txn_fold_whole_frame_t *frame) if (!ok) return false; } - if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-built-logged", frame->error)) - return false; - advanced.storage = storage; - advanced.folded = frame->point; - advanced.max_folded = frame->installed.next.max_folded; - advanced.base = frame->installed.next; - advanced.packs = frame->installed.packs; - advanced.built = true; - advanced.horizon = frame->horizon; - advanced.has_chain = false; - frame->installed.some = false; - txn_record_BaseAdvanced_encode(&advanced, &body); - ok = host_wal_insert(doors, HOST_WAL_KIND_BASE_ADVANCED, body.ptr, body.len, &lsn, frame->error); - std_vec_free_Checked(&body, TXN_FOLD_BYTES); - txn_record_BaseAdvanced_drop(&advanced); - return ok; + return host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-built-logged", frame->error); } -/* txn_fold_whole_body takes the writer's logged nodes, installs the base under the registration, and logs it. */ +/* + * txn_fold_whole_body takes the writer's logged nodes and logs them, then + * installs and logs the base under the registration. + */ static void txn_fold_whole_body(void *frame_) { @@ -5750,6 +6006,8 @@ txn_fold_whole_body(void *frame_) trees_store_Writer_logged(&frame->writer->core, &frame->logged); frame->logged_live = true; + if (!txn_fold_whole_logged(frame)) + return; if (!chunks_packs_roots_RootSlots_register(frame->io, txn_fold_whole_closure_0, frame, NULL, &frame->installed, &failed)) { @@ -5757,12 +6015,7 @@ txn_fold_whole_body(void *frame_) return; } *frame->result = frame->installed.some; - if (!frame->installed.some) - { - frame->ok = true; - return; - } - frame->ok = txn_fold_whole_logged(frame); + frame->ok = true; } /* txn_fold_whole_pad frees what whole holds live: the install, the logged nodes, the base, and the writer. */ @@ -5786,8 +6039,8 @@ txn_fold_whole_pad(void *frame_) * txn_fold_whole installs a base a backend wrote whole, taking the base and * the writer whose packs hold its trees' roots, held in the registration's * slots until the base that names them is installed as the state's newest, - * at a fold point, when its base generation is still the one read before the - * build, and for a permanent storage logged as built, after every node the + * at a fold point and a read point, when its base generation is still the + * one read before the build, and for a permanent storage logged as built, after every node the * writer kept as a Chunk record, each in its own checkpoint window, children * before parents; with nodes none it logs no chunk. The record carries its * replay's conflict horizon. It answers whether it installed. @@ -5795,13 +6048,13 @@ txn_fold_whole_pad(void *frame_) static bool txn_fold_whole(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, TreesStoreBrandedWriter *writer, const bool *has_roots, const size_t *roots, TxnBaseBase *base, - HostLsn point, uint64 generation, uint64 horizon, bool has_nodes, uint64 nodes, bool *result, - HostError *error) + HostLsn point, HostLsn read, uint64 generation, uint64 horizon, bool has_nodes, uint64 nodes, + bool *result, HostError *error) { txn_fold_whole_frame_t frame = {.io = io, .repo = repo, .reading = reading, .held = held, .writer = writer, .writer_live = true, .has_roots = has_roots, .roots = roots, .base = *base, .base_live = true, - .point = point, .generation = generation, .horizon = horizon, .has_nodes = has_nodes, .nodes = nodes, - .result = result, .error = error}; + .point = point, .read = read, .generation = generation, .horizon = horizon, .has_nodes = has_nodes, + .nodes = nodes, .result = result, .error = error}; std_landing(&frame, txn_fold_whole_body, txn_fold_whole_pad); txn_fold_whole_pad(&frame); @@ -5965,7 +6218,7 @@ txn_fold_built_body(void *frame_) } frame->writer_live = frame->base_live = false; frame->ok = txn_fold_whole(frame->io, frame->repo, frame->reading, frame->held, &frame->writer, frame->has_roots, - frame->roots, &frame->base, frame->point, generation, frame->horizon, + frame->roots, &frame->base, frame->point, frame->point, generation, frame->horizon, frame->has_nodes, frame->nodes, frame->result, frame->error); } @@ -6277,33 +6530,26 @@ txn_fold_reformed_body(void *frame_) TreesBuildEntry *entry = &row->entry; ChunksTrailerStamp stamp; uint32 heap_bytes; + uint64 copies_bytes = 0; txn_TupleU64VecU8U32_t pair; TreesMutateEdit edit = {.tag = TREES_MUTATE_EDIT_PUT}; - if (entry->stamps.len != 1) - { - *frame->error = txn_fold_fork("XX001", "a keyed table's entry holds other than one row"); - return; - } - stamp = STD_VEC_AT(&entry->stamps, ChunksTrailerStamp, 0); - if (row->epoch == end) + if (row->epoch != end) { - if (!txn_fold_written_heap(&frame->ctids, stamp.tuple, &heap_bytes)) + size_t from; + + if (entry->stamps.len != 1) { - txn_fold_reformed_corrupt(frame); + *frame->error = txn_fold_fork("XX001", "a keyed table's entry holds other than one row"); return; } - } - else - { - size_t from; - if (row->epoch < frame->base->epoch || row->epoch - frame->base->epoch > frame->chain_len) { txn_fold_reformed_corrupt(frame); return; } from = row->epoch - frame->base->epoch; + stamp = STD_VEC_AT(&entry->stamps, ChunksTrailerStamp, 0); if (!frame->reform->ops->reformed(frame->reform->self, frame->io, entry->key.ptr, entry->key.len, stamp, entry->value.ptr, entry->value.len, entry->addresses.ptr, entry->addresses.len, entry->values.ptr, entry->values.len, @@ -6318,14 +6564,25 @@ txn_fold_reformed_body(void *frame_) entry->values = frame->reformed.values; heap_bytes = frame->reformed.heap_bytes; } + for (size_t c = 0; c < entry->stamps.len; c++) + { + stamp = STD_VEC_AT(&entry->stamps, ChunksTrailerStamp, c); + if (row->epoch == end && !txn_fold_written_heap(&frame->ctids, stamp.tuple, &heap_bytes)) + { + txn_fold_reformed_corrupt(frame); + return; + } + copies_bytes += heap_bytes; + pair.f0 = stamp.tuple; + pair.f2 = heap_bytes; + std_vec_reserve_Checked(&frame->paired, 1, TXN_FOLD_PAIRED); + txn_fold_bytes(entry->key.ptr, entry->key.len, &pair.f1); + txn_fold_paired_insert(&frame->paired, &pair); + } *frame->sizes = txn_base_Sizes_summed(*frame->sizes, txn_fold_sized(entry->key.ptr, entry->key.len, - entry->value.ptr, entry->value.len, 1)); - frame->sizes->heap_bytes = STD_ADD(u64, frame->sizes->heap_bytes, heap_bytes, 1844, 9); - pair.f0 = stamp.tuple; - pair.f2 = heap_bytes; - std_vec_reserve_Checked(&frame->paired, 1, TXN_FOLD_PAIRED); - txn_fold_bytes(entry->key.ptr, entry->key.len, &pair.f1); - txn_fold_paired_insert(&frame->paired, &pair); + entry->value.ptr, entry->value.len, + entry->stamps.len)); + frame->sizes->heap_bytes = STD_ADD(u64, frame->sizes->heap_bytes, copies_bytes, 1844, 9); edit.u.put = *entry; std_vec_reserve_Checked(&frame->out, 1, TXN_FOLD_EDIT); std_vec_push_Checked(&frame->out, &edit, TXN_FOLD_EDIT); @@ -6408,8 +6665,11 @@ txn_fold_reformed_pad(void *frame_) * is stored at, through the caller's re-form, its tuple id paired again with * its new heap bytes, a row written at the chain's end kept as it is, and * the base's sizes counted afresh over every row it keeps. A keyed table's - * entry is one row with one stamp. It answers the edits, the tuple id tree's - * edits, the sizes, and the chain's end as the base's epoch. + * entry is one row with one stamp. A keyless table's rows are all at the + * chain's end, since its DDL re-keys every row at its commit, so each entry + * is kept with every copy's tuple id paired (bugs row 328). It answers the + * edits, the tuple id tree's edits, the sizes, and the chain's end as the + * base's epoch. */ static bool txn_fold_reformed(HostIoIo *io, TreesStoreNodeStore store, const TxnBaseBase *base, const TxnVisibleLive *chain, @@ -6610,6 +6870,8 @@ typedef struct txn_fold_copied_frame_t TxnBaseBase base; bool base_live; HostLsn point; + /* read is the installed base's read point. */ + HostLsn read; bool has_nodes; uint64 nodes; TxnFoldCopies *memo; @@ -6671,8 +6933,8 @@ txn_fold_copied_body(void *frame_) } frame->writer_live = frame->base_live = false; frame->ok = txn_fold_whole(frame->io, frame->repo, frame->reading, frame->held, &frame->writer, has_roots, roots, - &frame->base, frame->point, generation, frame->base.max_folded, frame->has_nodes, - frame->nodes, frame->result, frame->error); + &frame->base, frame->point, frame->read, generation, frame->base.max_folded, + frame->has_nodes, frame->nodes, frame->result, frame->error); } /* txn_fold_copied_pad frees what copied holds live: its writer and the base. */ @@ -6692,20 +6954,21 @@ txn_fold_copied_pad(void *frame_) * txn_fold_copied installs a copy of a base in a new storage, taking the * base, its four trees written again chunk by chunk into packs of the * storage's tablespace, every chunk they reach that a pack holds, each - * keeping its address, hashed bytes, and stamps, at a fold point, and logged - * as a built base whose replay's conflict horizon is its maxFolded. A chunk + * keeping its address, hashed bytes, and stamps, at a fold point and a read + * point, and logged as a built base whose replay's conflict horizon is its + * maxFolded. A chunk * the working state's delta holds keeps its unknown locator. A memo of the * chunks copied writes a chunk the base names more than once once. It * answers whether it installed. */ bool txn_fold_copied(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, - TxnBaseBase *base, HostLsn point, bool has_nodes, uint64 nodes, TxnFoldCopies *memo, bool *result, - HostError *error) + TxnBaseBase *base, HostLsn point, HostLsn read, bool has_nodes, uint64 nodes, TxnFoldCopies *memo, + bool *result, HostError *error) { txn_fold_copied_frame_t frame = {.io = io, .repo = repo, .reading = reading, .held = held, .base = *base, - .base_live = true, .point = point, .has_nodes = has_nodes, .nodes = nodes, .memo = memo, .result = result, - .error = error}; + .base_live = true, .point = point, .read = read, .has_nodes = has_nodes, .nodes = nodes, .memo = memo, + .result = result, .error = error}; std_landing(&frame, txn_fold_copied_body, txn_fold_copied_pad); txn_fold_copied_pad(&frame); @@ -6725,14 +6988,16 @@ txn_fold_installs_done(HostIoIo *io, const RepoRepo *repo, const HostSharedReade txn_fold_held_pairing_t paired = {TREES_STORE_READER_NODE_STORE(&reader), base->has_tuple_ids, base->tuple_ids}; TxnFoldSpaced spaced; - if (!txn_fold_dropped(io, held, base->folded, txn_fold_held_pairing, &paired, error) || - !txn_fold_spaced(io, held, base->folded, txn_fold_held_pairing, &paired, &spaced, error)) + if (!txn_fold_dropped(io, held, base->read, txn_fold_held_pairing, &paired, error) || + !txn_fold_spaced(io, held, base->read, NULL, NULL, &spaced, error)) return false; std_vec_drop_Checked(&spaced.edits, TXN_FOLD_EDIT, trees_mutate_Edit_drop); + std_vec_drop_Checked(&spaced.kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept), txn_fold_Kept_drop); if (!txn_fold_unspaced(io, held, &TXN_MEMTABLE_MAP_FREE_TREE_OPS, &spaced.removed, error) || - !txn_fold_valued(io, held, base->folded, &spaced, error)) + !txn_fold_valued(io, held, base->read, &spaced, error)) return false; std_vec_drop_Checked(&spaced.edits, TXN_FOLD_EDIT, trees_mutate_Edit_drop); + std_vec_drop_Checked(&spaced.kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept), txn_fold_Kept_drop); return txn_fold_unspaced(io, held, &TXN_MEMTABLE_MAP_VALUES_TREE_OPS, &spaced.removed, error) && txn_memtable_tuples_emptied(io, host_shared_Held_record(held), error); } @@ -7064,6 +7329,8 @@ typedef struct txn_fold_rebuilt_frame_t const HostSharedReader *reading; const HostSharedHeld *held; const TxnRecordBaseAdvanced *advanced; + /* end is the replayed record's end. */ + HostLsn end; uint64 generation; HostSharedHeld deltas; bool deltas_live; @@ -7144,7 +7411,10 @@ txn_fold_rebuilt_closure_2_body(void *frame_) if (rebuilt->has_roots[i]) *named[i] = trees_published_Published_base(&STD_VEC_AT(&frame->published, TreesPublishedPublished, at++)); } - if (!txn_state_installed(io, rebuilt->held, rebuilt->generation, next, &installed, &host)) + if (!chunks_packs_backend_Packs_flush(&rebuilt->repo->packs, io, rebuilt->reading, frame->error)) + return; + if (!txn_state_installed(io, rebuilt->held, rebuilt->generation, next, txn_fold_replayed, &rebuilt->end, + &installed, &host)) { *frame->error = chunks_error_Error_FromError_from(&host); return; @@ -7289,11 +7559,6 @@ txn_fold_rebuilt_body(void *frame_) frame->ok = true; return; } - if (!chunks_packs_backend_Packs_flush(&frame->repo->packs, frame->io, frame->reading, &failed)) - { - *frame->error = txn_base_storage(&failed); - return; - } for (size_t i = 0; i < frame->nodes.len; i++) { const txn_TupleAddressBool_t *node = &STD_VEC_AT(&frame->nodes, txn_TupleAddressBool_t, i); @@ -7334,14 +7599,16 @@ txn_fold_rebuilt_pad(void *frame_) * packs, children before parents, and installs the base that names them at * this server's locators, when its base generation is still the one read; * once the packs are flushed it takes those nodes out of the delta. It - * clones the record's key, since the installed base outlives the record. + * clones the record's key, since the installed base outlives the record. The + * installed base keeps the record's end. */ static bool txn_fold_rebuilt(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, - const TxnRecordBaseAdvanced *advanced, uint64 generation, bool *result, HostError *error) + const TxnRecordBaseAdvanced *advanced, HostLsn end, uint64 generation, bool *result, + HostError *error) { txn_fold_rebuilt_frame_t frame = {.io = io, .repo = repo, .reading = reading, .held = held, - .advanced = advanced, .generation = generation, .result = result, .error = error}; + .advanced = advanced, .end = end, .generation = generation, .result = result, .error = error}; std_landing(&frame, txn_fold_rebuilt_body, txn_fold_rebuilt_pad); txn_fold_rebuilt_pad(&frame); @@ -7376,6 +7643,45 @@ typedef struct txn_fold_redo_frame_t bool ok; } txn_fold_redo_frame_t; +/* + * txn_fold_redo_residuals lists, before a replayed fold's install, the + * residuals of the spilled map versions its drop keeps at its point, as its + * primary's fold listed them; a built base keeps none. + */ +static bool +txn_fold_redo_residuals(txn_fold_redo_frame_t *frame, StdVec *result) +{ + const TxnRecordBaseAdvanced *advanced = frame->advanced; + TxnFoldSpaced free; + TxnFoldSpaced values; + StdVec kept; + bool ok; + + if (advanced->built) + { + std_vec_new(result, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + return true; + } + if (!txn_fold_spaced(frame->io, frame->held, advanced->folded, NULL, NULL, &free, frame->error)) + return false; + if (!txn_fold_valued(frame->io, frame->held, advanced->folded, &values, frame->error)) + { + txn_fold_Spaced_drop(&free); + return false; + } + kept = free.kept; + std_vec_new(&free.kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept)); + std_vec_extend_from_slice_Checked(&kept, values.kept.ptr, values.kept.len, sizeof(TxnSpillKept), + _Alignof(TxnSpillKept)); + values.kept.len = 0; + txn_fold_Spaced_drop(&free); + txn_fold_Spaced_drop(&values); + ok = txn_spill_residual(frame->io, host_shared_Held_record(frame->held), advanced->folded, kept.ptr, kept.len, + result, frame->error); + std_vec_drop_Checked(&kept, sizeof(TxnSpillKept), _Alignof(TxnSpillKept), txn_fold_Kept_drop); + return ok; +} + /* txn_fold_redo_body installs the record's base, rebuilds a built one, or records the point, unless a newer base passed it. */ static void txn_fold_redo_body(void *frame_) @@ -7403,6 +7709,7 @@ txn_fold_redo_body(void *frame_) uint64 generation = txn_state_generation(host_shared_Held_record(frame->held)); bool passed = false; bool installed; + StdVec residuals; if (!txn_state_newest(frame->io, frame->held, &frame->newest_live, &frame->newest, frame->error)) return; @@ -7423,7 +7730,8 @@ txn_fold_redo_body(void *frame_) { bool rebuilt; - if (!txn_fold_rebuilt(frame->io, frame->repo, frame->reading, frame->held, advanced, generation, &rebuilt, + if (!txn_fold_rebuilt(frame->io, frame->repo, frame->reading, frame->held, advanced, frame->end, generation, + &rebuilt, frame->error)) return; if (rebuilt) @@ -7433,11 +7741,26 @@ txn_fold_redo_body(void *frame_) } continue; } + if (!txn_fold_redo_residuals(frame, &residuals)) + return; if (!txn_state_installed(HOST_IO_IO_TAKES_SHORT(frame->io), frame->held, generation, &advanced->base, + txn_fold_replayed, &frame->end, &installed, frame->error)) + { + std_vec_free_Checked(&residuals, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); return; + } if (!installed) + { + bool unlisted = txn_spill_unlisted(frame->io, host_shared_Held_record(frame->held), &residuals, + frame->error); + + std_vec_free_Checked(&residuals, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (!unlisted) + return; continue; + } + std_vec_free_Checked(&residuals, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); if (advanced->built) { txn_state_grown(host_shared_Held_record(frame->held), &advanced->base); diff --git a/src/doltgres/txn/c/identity.c b/src/doltgres/txn/c/identity.c index 8cec866028842..fa7d062e981fb 100644 --- a/src/doltgres/txn/c/identity.c +++ b/src/doltgres/txn/c/identity.c @@ -40,9 +40,20 @@ #include "doltgres/txn/memtable/branch.h" #include "doltgres/txn/memtable/table.h" #include "doltgres/txn/outcome.h" +#include "doltgres/txn/spill.h" #include "doltgres/txn/visible.h" #include "doltgres/txn/written.h" +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/chunks/record.h" +#include "doltgres/repo/lib.h" +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" + +/* RUST_FILE is the file the tests' asserts name, identity.rs. */ +#define RUST_FILE "txn" STD_SEP "src" STD_SEP "identity.rs" +#endif + /* The prepare whose closure pushes an identity change's versions, or answers false to start over. */ #define TXN_MEMTABLE_BRANCH_RS Bool #define TXN_MEMTABLE_BRANCH_WS Striping @@ -94,27 +105,14 @@ #define TXN_IDENTITY_OP sizeof(TxnMemtableOp), _Alignof(TxnMemtableOp) #define TXN_IDENTITY_NEED sizeof(txn_TupleRefSliceU8Usize_t), _Alignof(txn_TupleRefSliceU8Usize_t) #define TXN_IDENTITY_HANDLE sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle) -#define TXN_IDENTITY_FOLDED_VERSION sizeof(txn_TupleHandleOptionU64_t), _Alignof(txn_TupleHandleOptionU64_t) -#define TXN_IDENTITY_FOLDED sizeof(txn_TupleVecU8VecTupleHandleOptionU64_t), \ - _Alignof(txn_TupleVecU8VecTupleHandleOptionU64_t) +#define TXN_IDENTITY_FOLDED sizeof(txn_TupleVecU8VecHandle_t), _Alignof(txn_TupleVecU8VecHandle_t) -/* txn_TupleHandleOptionU64_t is dropped's (Handle, Option): a folded version and its top-level xid, or none. */ -typedef struct txn_TupleHandleOptionU64_t -{ - TxnMemtableVersionHandle f0; - bool has_f1; - uint64 f1; -} txn_TupleHandleOptionU64_t; - -/* - * txn_TupleVecU8VecTupleHandleOptionU64_t is dropped's (Vec, Vec<(Handle, - * Option)>): a key and its folded versions. - */ -typedef struct txn_TupleVecU8VecTupleHandleOptionU64_t +/* txn_TupleVecU8VecHandle_t is dropped's (Vec, Vec): a key and its folded versions. */ +typedef struct txn_TupleVecU8VecHandle_t { StdVec f0; StdVec f1; -} txn_TupleVecU8VecTupleHandleOptionU64_t; +} txn_TupleVecU8VecHandle_t; /* TxnIdentityFoundTag is Found's variants. */ typedef enum TxnIdentityFoundTag @@ -466,15 +464,6 @@ txn_identity_corrupt(void) return txn_identity_fork("XX001", "an identity tree's value has the wrong form"); } -/* txn_identity_storage returns what names a database's identity tree in a transaction's list of touched keys. */ -static HostStorage -txn_identity_storage(uint32 repository) -{ - HostStorage storage = {.database = repository}; - - return storage; -} - /* * txn_identity_stripe returns the key lock's stripe of a database's identity * tree, which every change of its pairs takes, so a derivation's probe and @@ -483,7 +472,7 @@ txn_identity_storage(uint32 repository) static uint32 txn_identity_stripe(uint32 repository) { - HostStorage storage = txn_identity_storage(repository); + HostStorage storage = txn_spill_identity_storage(repository); return txn_change_stripe(&storage, NULL, 0, false); } @@ -513,11 +502,42 @@ typedef struct txn_identity_found_frame_t bool ok; } txn_identity_found_frame_t; +/* txn_identity_found_dropped drops the key's versions a read of found gave. */ +static void +txn_identity_found_dropped(void *captures) +{ + txn_identity_found_frame_t *frame = captures; + + if (frame->versions_live) + txn_memtable_version_versions_drop(&frame->versions, 0); + frame->versions_live = false; +} + +/* txn_identity_found_read is found's read: the key's versions in the memtable, then in its spills (bugs row 291). */ +static bool +txn_identity_found_read(void *captures, uint64 *listed, HostError *error) +{ + txn_identity_found_frame_t *frame = captures; + StdVec spilled; + + if (!txn_memtable_branch_identity_versions(frame->io, frame->hold, frame->deltas, frame->key, frame->key_len, + &frame->versions, error)) + return false; + frame->versions_live = true; + if (!txn_spill_versions(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, frame->hold, frame->deltas, + TXN_RUNS_KIND_SPILLED_IDENTITY, frame->key, frame->key_len, listed, &spilled, error)) + return false; + std_vec_extend_from_slice_Checked(&frame->versions, spilled.ptr, spilled.len, sizeof(TxnMemtableVersionVersion), + _Alignof(TxnMemtableVersionVersion)); + std_vec_free_Checked(&spilled, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + return true; +} + /* - * txn_identity_found_body reads the key's versions and then the folded - * tree's value, resolves the key, and for a key that exists answers the - * newest put whose writer is the caller's transaction or committed, or the - * folded tree's value. + * txn_identity_found_body reads the key's versions, the memtable's and then + * its spills' (bugs row 291), and then the folded tree's value, resolves the + * key, and for a key that exists answers the newest put whose writer is the + * caller's transaction or committed, or the folded tree's value. */ static void txn_identity_found_body(void *frame_) @@ -526,10 +546,9 @@ txn_identity_found_body(void *frame_) ChunksErrorError storage; StdBytes based; - if (!txn_memtable_branch_identity_versions(frame->io, frame->hold, frame->deltas, frame->key, frame->key_len, - &frame->versions, frame->error)) + if (!txn_spill_taken(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, frame->deltas, txn_identity_found_read, + txn_identity_found_dropped, frame, frame->error)) return; - frame->versions_live = true; if (!repo_Repo_identity(frame->repo, frame->io, frame->reading, frame->key, frame->key_len, &frame->has_based, &frame->based, &storage)) { @@ -612,8 +631,9 @@ txn_identity_found_pad(void *frame_) * reads its key, the caller's own changes and every committed one counted, * with the value of the version that decides it: the newest put whose writer * is the caller's transaction or committed, or the folded tree's value with - * no such version. It reads the delta's versions before the folded tree, so - * a save between the reads only folds versions it has already read. + * no such version. It reads the delta's versions, its spills' included, + * before the folded tree, so a save between the reads only folds versions it + * has already read. */ static bool txn_identity_found(HostIoIo *io, const HostDeltaHold *hold, const RepoRepo *repo, const HostSharedReader *reading, @@ -839,8 +859,8 @@ typedef struct txn_identity_changed_closure_4_frame_t /* * txn_identity_changed_closure_4_body takes the tree's stripe, answers false * when another writer pushed under it since the caller's count, and else in - * one checkpoint window appends each change's record and pushes its version - * with its LSN. + * one checkpoint window appends each of the prepare's group's changes' records + * and pushes its version with its LSN. */ static void txn_identity_changed_closure_4_body(void *frame_) @@ -865,8 +885,9 @@ txn_identity_changed_closure_4_body(void *frame_) if (!host_delay_checkpoint(HOST_IO_TAKES_SHORT_IO_DOORS(io), NULL, 0, &frame->window, frame->error)) return; frame->window_live = true; - for (size_t at = 0; at < changed->bodies.len; at++) + for (size_t i = 0; i < frame->prepared->aims_len; i++) { + size_t at = frame->prepared->first + i; const StdVec *body = &STD_VEC_AT(&changed->bodies, StdVec, at); TxnMemtableVersionChange change = {.writer = changed->writer, .op = &STD_VEC_AT(&changed->ops, TxnMemtableOp, at), .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, @@ -875,7 +896,7 @@ txn_identity_changed_closure_4_body(void *frame_) if (!host_wal_insert(HOST_IO_TAKES_SHORT_IO_DOORS(io), HOST_WAL_KIND_DATABASE_CHANGE, body->ptr, body->len, &change.lsn, frame->error) || - !txn_memtable_branch_identity_finish(io, frame->prepared, at, &change, true, changed->roomed, &handle, + !txn_memtable_branch_identity_finish(io, frame->prepared, i, &change, true, changed->roomed, &handle, frame->error)) return; } @@ -912,15 +933,16 @@ txn_identity_changed_closure_4(void *captures, HostIoIo *io, TxnMemtableTablePre } /* - * txn_identity_changed_body records each pair's key in the transaction's - * list and makes its record, key, and op, then the versions' needs, and - * prepares the push. + * txn_identity_changed_body makes each pair's record, key, and op, then + * records each key in the transaction's list with its version's need, + * spilling the transaction's own identity versions past the spill threshold + * (bugs row 291), and prepares the push. */ static void txn_identity_changed_body(void *frame_) { txn_identity_changed_frame_t *frame = frame_; - HostStorage storage = txn_identity_storage(frame->repository); + HostStorage storage = txn_spill_identity_storage(frame->repository); uint64 top = frame->writer.has_top ? frame->writer.top : frame->writer.xid; TxnMemtableVersionShown command = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; size_t len = frame->pairs.len; @@ -949,13 +971,9 @@ txn_identity_changed_body(void *frame_) TxnRecordDatabaseChange change = {.tree = TXN_RECORD_TREE_IDENTITY, .writer = frame->writer, .repository = frame->repository, .key = pair->f0, .has_value = pair->has_f1, .value = pair->f1}; - uint64 touched; StdVec body; TxnMemtableOp op; - if (!txn_written_touched(frame->io, top, &storage, TXN_WRITTEN_OF_IDENTITY, pair->f0.ptr, pair->f0.len, 0, - &touched, frame->error)) - return; txn_record_DatabaseChange_encode(&change, &body); std_vec_push_Checked(&frame->bodies, &body, TXN_IDENTITY_VEC); std_vec_push_Checked(&frame->keys, &change.key, TXN_IDENTITY_VEC); @@ -969,7 +987,18 @@ txn_identity_changed_body(void *frame_) const StdVec *key = &STD_VEC_AT(&frame->keys, StdVec, i); txn_TupleRefSliceU8Usize_t need = {{key->ptr, key->len}, txn_memtable_version_encoded(&frame->writer, &STD_VEC_AT(&frame->ops, TxnMemtableOp, i), command, true, 0)}; + uint64 touched; + if (!txn_written_touched(frame->io, top, &storage, TXN_WRITTEN_OF_IDENTITY, key->ptr, key->len, need.f1, + &touched, frame->error)) + return; + if (touched >= txn_spill_threshold()) + { + bool spilled; + + if (!txn_spill_spilled(frame->io, &storage, TXN_WRITTEN_OF_IDENTITY, top, &spilled, frame->error)) + return; + } std_vec_push_Checked(&frame->needs, &need, TXN_IDENTITY_NEED); } frame->ok = txn_memtable_branch_identity_prepare_Bool_Striping(frame->io, host_shared_Held_record(frame->deltas), @@ -1003,11 +1032,14 @@ txn_identity_changed_pad(void *frame_) * txn_identity_changed gives keys of a database's identity tree their * values, a removal for none, in the running transaction, taking the pairs: * it records each key in the transaction's list, which an abort prunes, then - * in one prepare's window under the tree's stripe appends each change's - * record and pushes its version with its LSN, every pair of a database being + * in the window of each of the prepare's groups under the tree's stripe + * appends each change's record and pushes its version with its LSN (bugs row + * 338), every pair of a database being * logged. With the push count the caller read before its probe, it pushes * nothing and answers false when another writer pushed under the stripe - * since. + * since. A change with a count is at most TXN_MEMTABLE_TABLE_WHOLE pairs, one + * group, since a later group's check would meet the earlier groups' own + * pushes and start them over. */ static bool txn_identity_changed(HostIoIo *io, HostIoWaits *waits, const HostSharedHeld *deltas, uint32 repository, @@ -1018,6 +1050,7 @@ txn_identity_changed(HostIoIo *io, HostIoWaits *waits, const HostSharedHeld *del .writer = writer, .roomed = roomed, .pairs = *pairs, .has_count = has_count, .count = count, .result = result, .error = error}; + Assert(!has_count || pairs->len <= TXN_MEMTABLE_TABLE_WHOLE); std_landing(&frame, txn_identity_changed_body, txn_identity_changed_pad); txn_identity_changed_pad(&frame); return frame.ok; @@ -1314,18 +1347,64 @@ txn_identity_removed(HostIoIo *io, HostIoWaits *waits, uint32 repository, uint32 return frame.ok; } -/* txn_identity_folded_closure_0_captures_t is folded's walk's captures: the deltas, the fold point, and the pairs. */ +/* txn_identity_folded_closure_0_captures_t is folded's walk's captures: the deltas, the fold point, the pairs, and the keys the walk read, in order. */ typedef struct txn_identity_folded_closure_0_captures_t { HostSharedRecord deltas; HostLsn point; StdVec *pairs; + StdVec *walked; } txn_identity_folded_closure_0_captures_t; +/* + * txn_identity_folded_key adds a key's change of its newest version, from + * at, that the fold applies, if any, dropping each version it passes over. + */ +static bool +txn_identity_folded_key(HostIoIo *io, HostLsn point, const uint8 *key, size_t key_len, StdVec *versions, size_t *at, + StdVec *pairs, HostError *error) +{ + for (; *at < versions->len; (*at)++) + { + TxnMemtableVersionVersion *version = &STD_VEC_AT(versions, TxnMemtableVersionVersion, *at); + TxnFoldApplied applied; + TxnMemtableVersionRecord record; + TxnMemtableVersionOpRef op; + TxnIdentityPair pair = {0}; + + if (!txn_fold_applied(io, version, point, &applied, error)) + return false; + if (applied.tag != TXN_FOLD_APPLIED_YES) + { + txn_memtable_version_Version_drop(version); + continue; + } + record = txn_memtable_version_Version_record(version); + op = txn_memtable_version_Record_op(&record); + if (op.tag != TXN_MEMTABLE_VERSION_OP_REF_PUT && op.tag != TXN_MEMTABLE_VERSION_OP_REF_DELETE) + { + *error = txn_identity_fork("XX001", "an identity tree's version neither puts nor deletes"); + return false; + } + if (!std_vec_try_reserve(pairs, 1, TXN_IDENTITY_PAIR)) + { + *error = host_Error_out_of_memory(); + return false; + } + pair.has_f1 = op.tag == TXN_MEMTABLE_VERSION_OP_REF_PUT; + if (pair.has_f1) + txn_identity_bytes(op.u.put.value.ptr, op.u.put.value.len, &pair.f1); + txn_identity_bytes(key, key_len, &pair.f0); + std_vec_push_Checked(pairs, &pair, TXN_IDENTITY_PAIR); + return true; + } + return true; +} + /* * txn_identity_folded_closure_0_frame_t is the walk's state across its pad: - * a key's hold, its versions from the first it hasn't dropped, and the value - * it found, each while live. + * a key's hold and its versions from the first it hasn't dropped, each while + * live. */ typedef struct txn_identity_folded_closure_0_frame_t { @@ -1338,85 +1417,77 @@ typedef struct txn_identity_folded_closure_0_frame_t StdVec versions; size_t at; bool versions_live; - bool has_value; - StdVec value; - bool value_live; HostError *error; bool ok; } txn_identity_folded_closure_0_frame_t; -/* txn_identity_folded_closure_0_body adds the key's change of its newest version the fold applies, if any. */ +/* txn_identity_folded_closure_0_dropped drops the key's versions a read gave. */ +static void +txn_identity_folded_closure_0_dropped(void *captures) +{ + txn_identity_folded_closure_0_frame_t *frame = captures; + + if (frame->versions_live) + txn_memtable_version_versions_drop(&frame->versions, 0); + frame->versions_live = false; +} + +/* txn_identity_folded_closure_0_read reads the key's memtable versions, then its spills' below them. */ +static bool +txn_identity_folded_closure_0_read(void *captures, uint64 *listed, HostError *error) +{ + txn_identity_folded_closure_0_frame_t *frame = captures; + StdVec spilled; + + if (!txn_memtable_branch_identity_versions(frame->io, &frame->hold, frame->walk->deltas, frame->key, + frame->key_len, &frame->versions, error)) + return false; + frame->versions_live = true; + if (!txn_spill_versions(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, &frame->hold, frame->walk->deltas, + TXN_RUNS_KIND_SPILLED_IDENTITY, frame->key, frame->key_len, listed, &spilled, error)) + return false; + std_vec_extend_from_slice_Checked(&frame->versions, spilled.ptr, spilled.len, sizeof(TxnMemtableVersionVersion), + _Alignof(TxnMemtableVersionVersion)); + std_vec_free_Checked(&spilled, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + return true; +} + +/* txn_identity_folded_closure_0_body reads the key's memtable versions, then its spills' below them, keeps the key as walked, and adds its change. */ static void txn_identity_folded_closure_0_body(void *frame_) { txn_identity_folded_closure_0_frame_t *frame = frame_; txn_identity_folded_closure_0_captures_t *walk = frame->walk; + StdVec key; if (!host_delta_hold(frame->io, &frame->hold, frame->error)) return; frame->hold_live = true; - if (!txn_memtable_branch_identity_versions(frame->io, &frame->hold, walk->deltas, frame->key, frame->key_len, - &frame->versions, frame->error)) + if (!txn_spill_taken(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, walk->deltas, txn_identity_folded_closure_0_read, + txn_identity_folded_closure_0_dropped, frame, frame->error)) return; - frame->versions_live = true; - for (; frame->at < frame->versions.len; frame->at++) + if (!std_vec_try_reserve(walk->walked, 1, TXN_IDENTITY_VEC)) { - TxnMemtableVersionVersion *version = &STD_VEC_AT(&frame->versions, TxnMemtableVersionVersion, frame->at); - TxnFoldApplied applied; - TxnMemtableVersionRecord record; - TxnMemtableVersionOpRef op; - TxnIdentityPair pair; - - if (!txn_fold_applied(frame->io, version, walk->point, &applied, frame->error)) - return; - if (applied.tag != TXN_FOLD_APPLIED_YES) - { - txn_memtable_version_Version_drop(version); - continue; - } - record = txn_memtable_version_Version_record(version); - op = txn_memtable_version_Record_op(&record); - switch (op.tag) - { - case TXN_MEMTABLE_VERSION_OP_REF_PUT: - txn_identity_bytes(op.u.put.value.ptr, op.u.put.value.len, &frame->value); - frame->has_value = frame->value_live = true; - break; - case TXN_MEMTABLE_VERSION_OP_REF_DELETE: - break; - default: - *frame->error = txn_identity_fork("XX001", "an identity tree's version neither puts nor deletes"); - return; - } - if (!std_vec_try_reserve(walk->pairs, 1, TXN_IDENTITY_PAIR)) - { - *frame->error = host_Error_out_of_memory(); - return; - } - txn_identity_bytes(frame->key, frame->key_len, &pair.f0); - pair.has_f1 = frame->has_value; - if (frame->has_value) - pair.f1 = frame->value; - std_vec_push_Checked(walk->pairs, &pair, TXN_IDENTITY_PAIR); - frame->value_live = false; - break; + *frame->error = host_Error_out_of_memory(); + return; } - frame->ok = true; + txn_identity_bytes(frame->key, frame->key_len, &key); + std_vec_push_Checked(walk->walked, &key, TXN_IDENTITY_VEC); + frame->ok = txn_identity_folded_key(frame->io, walk->point, frame->key, frame->key_len, &frame->versions, + &frame->at, walk->pairs, frame->error); } -/* txn_identity_folded_closure_0_pad frees what the walk holds live: its value, its versions, and its hold. */ +/* txn_identity_folded_closure_0_pad frees what the walk holds live: its versions and its hold. */ static void txn_identity_folded_closure_0_pad(void *frame_) { txn_identity_folded_closure_0_frame_t *frame = frame_; - if (frame->value_live) - std_vec_free_Checked(&frame->value, TXN_IDENTITY_BYTES); if (frame->versions_live) txn_memtable_version_versions_drop(&frame->versions, frame->at); if (frame->hold_live) host_delta_Hold_drop(&frame->hold); - frame->value_live = frame->versions_live = frame->hold_live = false; + frame->versions_live = frame->hold_live = false; } /* txn_identity_folded_closure_0 is folded's walk's closure, of one key of the delta. */ @@ -1432,7 +1503,11 @@ txn_identity_folded_closure_0(void *captures, HostIoIo *io, const uint8 *key, si return frame.ok; } -/* txn_identity_folded_frame_t is folded's state across its pad: its deltas and its pairs, each while live. */ +/* + * txn_identity_folded_frame_t is folded's state across its pad: its deltas, + * its pairs, the keys its walk read, and the spills' next key, its versions, + * and its hold, each while live. + */ typedef struct txn_identity_folded_frame_t { HostIoIo *io; @@ -1442,18 +1517,44 @@ typedef struct txn_identity_folded_frame_t bool deltas_live; StdVec pairs; bool pairs_live; + StdVec walked; + bool walked_live; + HostDeltaHold hold; + bool hold_live; + bool has_after; + StdVec after; + txn_TupleVecU8VecVersion_t next; + size_t at; + bool next_live; StdVec *result; HostError *error; bool ok; } txn_identity_folded_frame_t; -/* txn_identity_folded_body walks the tree's delta, if the database has one, for each key's folded change. */ +/* txn_identity_folded_walked_cmp orders a key among the walked keys by its bytes. */ +static int +txn_identity_folded_walked_cmp(const void *a, const void *b) +{ + const StdVec *x = a; + const StdVec *y = b; + + return std_bytes_Ord_cmp(x->ptr, x->len, y->ptr, y->len); +} + +/* + * txn_identity_folded_body walks the tree's delta, if the database has one, + * for each key's folded change, and then, when a listed identity spill's + * transaction is one the point applies, the keys its identity spills alone + * hold (bugs row 291). + */ static void txn_identity_folded_body(void *frame_) { txn_identity_folded_frame_t *frame = frame_; - txn_identity_folded_closure_0_captures_t walk = {.point = frame->point, .pairs = &frame->pairs}; + txn_identity_folded_closure_0_captures_t walk = {.point = frame->point, .pairs = &frame->pairs, + .walked = &frame->walked}; bool some; + bool applies; if (!txn_memtable_branch_find(HOST_SHARED_IO_WITHIN(frame->io), frame->repository, &some, &frame->deltas, frame->error)) @@ -1467,33 +1568,78 @@ txn_identity_folded_body(void *frame_) } frame->deltas_live = true; frame->pairs_live = true; + std_vec_new(&frame->walked, TXN_IDENTITY_VEC); + frame->walked_live = true; walk.deltas = host_shared_Held_record(&frame->deltas); if (!txn_memtable_table_walked(&TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS, frame->io, walk.deltas, txn_identity_folded_closure_0, &walk, NULL, frame->error)) return; + if (!txn_spill_identity_applies(frame->io, walk.deltas, frame->point, &applies, frame->error)) + return; + qsort(frame->walked.ptr, frame->walked.len, sizeof(StdVec), txn_identity_folded_walked_cmp); + while (applies) + { + if (!host_delta_hold(frame->io, &frame->hold, frame->error)) + return; + frame->hold_live = true; + if (!txn_spill_next(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, &frame->hold, walk.deltas, + TXN_RUNS_KIND_SPILLED_IDENTITY, frame->has_after, frame->after.ptr, frame->after.len, + TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, NULL, &some, &frame->next, frame->error)) + return; + if (!some) + break; + frame->next_live = true; + frame->at = 0; + if (bsearch(&frame->next.f0, frame->walked.ptr, frame->walked.len, sizeof(StdVec), + txn_identity_folded_walked_cmp) == NULL && + !txn_identity_folded_key(frame->io, frame->point, frame->next.f0.ptr, frame->next.f0.len, + &frame->next.f1, &frame->at, &frame->pairs, frame->error)) + return; + if (frame->has_after) + std_vec_free_Checked(&frame->after, TXN_IDENTITY_BYTES); + frame->after = frame->next.f0; + frame->has_after = true; + txn_memtable_version_versions_drop(&frame->next.f1, frame->at); + frame->next_live = false; + frame->hold_live = false; + host_delta_Hold_drop(&frame->hold); + } *frame->result = frame->pairs; frame->pairs_live = false; frame->ok = true; } -/* txn_identity_folded_pad frees what folded holds live: its pairs and its deltas. */ +/* txn_identity_folded_pad frees what folded holds live: the spills' next key and its hold, the walked keys, its pairs, and its deltas. */ static void txn_identity_folded_pad(void *frame_) { txn_identity_folded_frame_t *frame = frame_; + if (frame->next_live) + { + txn_memtable_version_versions_drop(&frame->next.f1, frame->at); + std_vec_free_Checked(&frame->next.f0, TXN_IDENTITY_BYTES); + } + if (frame->has_after) + std_vec_free_Checked(&frame->after, TXN_IDENTITY_BYTES); + if (frame->hold_live) + host_delta_Hold_drop(&frame->hold); + if (frame->walked_live) + std_vec_drop_Checked(&frame->walked, TXN_IDENTITY_VEC, txn_identity_bytes_drop); if (frame->pairs_live) std_vec_drop_Checked(&frame->pairs, TXN_IDENTITY_PAIR, txn_identity_Pair_drop); if (frame->deltas_live) host_shared_Held_drop(&frame->deltas); - frame->pairs_live = frame->deltas_live = false; + frame->next_live = frame->has_after = frame->hold_live = frame->walked_live = frame->pairs_live = + frame->deltas_live = false; } /* * txn_identity_folded returns what a save at a fold point folds of a - * database's identity tree: for each key of the tree's delta, the change of - * its newest version the fold applies, as the key's value or none for its - * removal, which the save makes only where the folded tree holds another. + * database's identity tree: for each key of the tree's delta and its spills, + * the change of its newest version the fold applies, as the key's value or + * none for its removal, which the save makes only where the folded tree holds + * another. */ bool txn_identity_folded(HostIoIo *io, uint32 repository, HostLsn point, StdVec *result, HostError *error) @@ -1510,10 +1656,10 @@ txn_identity_folded(HostIoIo *io, uint32 repository, HostLsn point, StdVec *resu static void txn_identity_folded_drop(void *folded_) { - txn_TupleVecU8VecTupleHandleOptionU64_t *folded = folded_; + txn_TupleVecU8VecHandle_t *folded = folded_; std_vec_free_Checked(&folded->f0, TXN_IDENTITY_BYTES); - std_vec_free_Checked(&folded->f1, TXN_IDENTITY_FOLDED_VERSION); + std_vec_free_Checked(&folded->f1, TXN_IDENTITY_HANDLE); } /* @@ -1558,12 +1704,12 @@ txn_identity_dropped_closure_0_body(void *frame_) { txn_identity_dropped_closure_0_frame_t *frame = frame_; txn_identity_dropped_closure_0_captures_t *walk = frame->walk; - txn_TupleVecU8VecTupleHandleOptionU64_t folded; + txn_TupleVecU8VecHandle_t folded; if (!host_delta_hold(frame->io, &frame->hold, frame->error)) return; frame->hold_live = true; - std_vec_new(&frame->applied, TXN_IDENTITY_FOLDED_VERSION); + std_vec_new(&frame->applied, TXN_IDENTITY_HANDLE); frame->applied_live = true; if (!txn_memtable_branch_identity_versions(frame->io, &frame->hold, walk->deltas, frame->key, frame->key_len, &frame->versions, frame->error)) @@ -1578,15 +1724,12 @@ txn_identity_dropped_closure_0_body(void *frame_) return; if (applied.tag == TXN_FOLD_APPLIED_YES) { - txn_TupleHandleOptionU64_t top = {.f0 = version->handle, .has_f1 = applied.u.yes.has_top, - .f1 = applied.u.yes.top}; - - if (!std_vec_try_reserve(&frame->applied, 1, TXN_IDENTITY_FOLDED_VERSION)) + if (!std_vec_try_reserve(&frame->applied, 1, TXN_IDENTITY_HANDLE)) { *frame->error = host_Error_out_of_memory(); return; } - std_vec_push_Checked(&frame->applied, &top, TXN_IDENTITY_FOLDED_VERSION); + std_vec_push_Checked(&frame->applied, &version->handle, TXN_IDENTITY_HANDLE); } txn_memtable_version_Version_drop(version); } @@ -1614,7 +1757,7 @@ txn_identity_dropped_closure_0_pad(void *frame_) if (frame->versions_live) txn_memtable_version_versions_drop(&frame->versions, frame->at); if (frame->applied_live) - std_vec_free_Checked(&frame->applied, TXN_IDENTITY_FOLDED_VERSION); + std_vec_free_Checked(&frame->applied, TXN_IDENTITY_HANDLE); if (frame->hold_live) host_delta_Hold_drop(&frame->hold); frame->versions_live = frame->applied_live = frame->hold_live = false; @@ -1648,18 +1791,15 @@ typedef struct txn_identity_dropped_frame_t StdVec folded; size_t at; bool folded_live; - StdVec handles; - bool handles_live; - StdVec taken; - bool taken_live; + bool *took; HostError *error; bool ok; } txn_identity_dropped_frame_t; /* * txn_identity_dropped_body lists each key's versions the fold applied, then - * takes each key's out of its chain and gives each taken one's count back at - * its top-level transaction's entry. + * takes each key's out of its chain, the remover giving back each counted + * one's count at its top-level transaction's entry. */ static void txn_identity_dropped_body(void *frame_) @@ -1667,7 +1807,9 @@ txn_identity_dropped_body(void *frame_) txn_identity_dropped_frame_t *frame = frame_; txn_identity_dropped_closure_0_captures_t walk = {.point = frame->point, .folded = &frame->folded}; bool some; + bool files; + *frame->took = false; if (!txn_memtable_branch_find(HOST_SHARED_IO_WITHIN(frame->io), frame->repository, &some, &frame->deltas, frame->error)) return; @@ -1683,68 +1825,37 @@ txn_identity_dropped_body(void *frame_) if (!txn_memtable_table_walked(&TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS, frame->io, walk.deltas, txn_identity_dropped_closure_0, &walk, NULL, frame->error)) return; + *frame->took = frame->folded.len > 0; for (; frame->at < frame->folded.len; frame->at++) { - txn_TupleVecU8VecTupleHandleOptionU64_t *key = &STD_VEC_AT(&frame->folded, - txn_TupleVecU8VecTupleHandleOptionU64_t, - frame->at); - size_t zipped; - - std_vec_with_capacity_Checked(&frame->handles, key->f1.len, TXN_IDENTITY_HANDLE); - frame->handles_live = true; - for (size_t i = 0; i < key->f1.len; i++) - std_vec_push_Checked(&frame->handles, &STD_VEC_AT(&key->f1, txn_TupleHandleOptionU64_t, i).f0, - TXN_IDENTITY_HANDLE); + txn_TupleVecU8VecHandle_t *key = &STD_VEC_AT(&frame->folded, txn_TupleVecU8VecHandle_t, frame->at); + StdVec taken; + if (!txn_memtable_table_removed(&TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS, frame->io, walk.deltas, key->f0.ptr, - key->f0.len, frame->handles.ptr, frame->handles.len, &frame->taken, - frame->error)) + key->f0.len, key->f1.ptr, key->f1.len, &taken, frame->error)) return; - frame->taken_live = true; - zipped = Min(key->f1.len, frame->taken.len); - for (size_t i = 0; i < zipped; i++) - { - const txn_TupleHandleOptionU64_t *top = &STD_VEC_AT(&key->f1, txn_TupleHandleOptionU64_t, i); - HostXid xid = {top->f1}; - bool has_added; - uint64 added; - - if (!STD_VEC_AT(&frame->taken, bool, i) || !top->has_f1) - continue; - if (!txn_status_add(frame->io, xid, PG_UINT64_MAX, &has_added, &added, frame->error)) - return; - } - frame->taken_live = false; - std_vec_free_Checked(&frame->taken, sizeof(bool), _Alignof(bool)); - frame->handles_live = false; - std_vec_free_Checked(&frame->handles, TXN_IDENTITY_HANDLE); + std_vec_free_Checked(&taken, sizeof(bool), _Alignof(bool)); txn_identity_folded_drop(key); } - frame->ok = true; + frame->ok = txn_spill_identity_folded(frame->io, walk.deltas, frame->point, &files, frame->error); + *frame->took = *frame->took || files; } -/* - * txn_identity_dropped_pad frees what dropped holds live: a key's removal and - * handles, the folded keys left, and its deltas. - */ +/* txn_identity_dropped_pad frees what dropped holds live: the folded keys left and its deltas. */ static void txn_identity_dropped_pad(void *frame_) { txn_identity_dropped_frame_t *frame = frame_; - if (frame->taken_live) - std_vec_free_Checked(&frame->taken, sizeof(bool), _Alignof(bool)); - if (frame->handles_live) - std_vec_free_Checked(&frame->handles, TXN_IDENTITY_HANDLE); if (frame->folded_live) { for (; frame->at < frame->folded.len; frame->at++) - txn_identity_folded_drop(&STD_VEC_AT(&frame->folded, txn_TupleVecU8VecTupleHandleOptionU64_t, - frame->at)); + txn_identity_folded_drop(&STD_VEC_AT(&frame->folded, txn_TupleVecU8VecHandle_t, frame->at)); std_vec_free_Checked(&frame->folded, TXN_IDENTITY_FOLDED); } if (frame->deltas_live) host_shared_Held_drop(&frame->deltas); - frame->taken_live = frame->handles_live = frame->folded_live = frame->deltas_live = false; + frame->folded_live = frame->deltas_live = false; } /* @@ -1754,12 +1865,14 @@ txn_identity_dropped_pad(void *frame_) * one's count back at its top-level transaction's entry, so the folded pairs * live in the manifest's tree alone. A reader reads the delta and then the * newest manifest (found), so no reader tells the two apart once the save is - * durable, and no horizon is waited for. + * durable, and no horizon is waited for. It answers whether it took a + * version or an identity spill out (bugs row 340). */ bool -txn_identity_dropped(HostIoIo *io, uint32 repository, HostLsn point, HostError *error) +txn_identity_dropped(HostIoIo *io, uint32 repository, HostLsn point, bool *took, HostError *error) { - txn_identity_dropped_frame_t frame = {.io = io, .repository = repository, .point = point, .error = error}; + txn_identity_dropped_frame_t frame = {.io = io, .repository = repository, .point = point, .took = took, + .error = error}; std_landing(&frame, txn_identity_dropped_body, txn_identity_dropped_pad); txn_identity_dropped_pad(&frame); @@ -1807,7 +1920,12 @@ typedef struct txn_identity_redo_frame_t bool ok; } txn_identity_redo_frame_t; -/* txn_identity_redo_body pushes the replayed change's version unless the delta holds it already. */ +/* + * txn_identity_redo_body pushes the replayed change's version unless the + * delta, its spills included, holds it already, counting it toward its + * transaction's replayed bytes, which spill past the threshold (bugs row + * 291). + */ static void txn_identity_redo_body(void *frame_) { @@ -1820,13 +1938,19 @@ txn_identity_redo_body(void *frame_) TxnMemtableVersionChange pushed; txn_TupleRefSliceU8Usize_t needs; TxnMemtableVersionHandle handle; + HostStorage storage = txn_spill_identity_storage(change->repository); + uint64 top = change->writer.has_top ? change->writer.top : change->writer.xid; + uint64 counted; + bool spilled; if (!txn_memtable_branch_deltas(frame->io, change->repository, frame->own, &frame->deltas, frame->error)) return; frame->deltas_live = true; deltas = host_shared_Held_record(&frame->deltas); if (!txn_memtable_branch_identity_holds(frame->io, deltas, change->key.ptr, change->key.len, frame->lsn, &holds, - frame->error)) + frame->error) || + (!holds && !txn_spill_holds(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, deltas, TXN_RUNS_KIND_SPILLED_IDENTITY, + change->key.ptr, change->key.len, frame->lsn, &holds, frame->error))) return; if (holds) { @@ -1841,6 +1965,10 @@ txn_identity_redo_body(void *frame_) needs.f0.ptr = change->key.ptr; needs.f0.len = change->key.len; needs.f1 = txn_memtable_version_encoded(&change->writer, &frame->op, replayed, true, frame->lsn); + if (!txn_spill_replayed(frame->io, &storage, TXN_WRITTEN_OF_IDENTITY, top, needs.f1, &counted, frame->error) || + (counted >= txn_spill_threshold() && + !txn_spill_spilled(frame->io, &storage, TXN_WRITTEN_OF_IDENTITY, top, &spilled, frame->error))) + return; pushed.writer = change->writer; pushed.op = &frame->op; pushed.lock = TXN_MEMTABLE_LOCK_MODE_UPDATE; @@ -1873,7 +2001,7 @@ txn_identity_redo_pad(void *frame_) * txn_identity_redo pushes a replayed identity change into its database's * working state's delta, taking it, in the startup process or on a standby, * with no catalog access, unless the checkpoint's runs brought it back, - * which the delta holds by its key and the record's LSN. + * which the delta or its spills hold by its key and the record's LSN. */ bool txn_identity_redo(HostIoIo *io, TxnRecordDatabaseChange *change, uint32 own, uint64 lsn, HostError *error) @@ -1885,3 +2013,862 @@ txn_identity_redo(HostIoIo *io, TxnRecordDatabaseChange *change, uint32 own, uin txn_identity_redo_pad(&frame); return frame.ok; } + +#ifdef DOLTGRES_TEST_TXN +/* TXN_IDENTITY_TESTS_UNWRAP is Result::unwrap's panic text. */ +#define TXN_IDENTITY_TESTS_UNWRAP "called `Result::unwrap()` on an `Err` value" + +/* TXN_IDENTITY_TESTS_SCHEMA is the parent id the tests' tables derive their ids under. */ +#define TXN_IDENTITY_TESTS_SCHEMA (TXN_IDENTITY_OBJECT_ID_BIT | 5) + +/* txn_identity_tests_ok unwraps a host result at a line. */ +static void +txn_identity_tests_ok(bool ok, HostError *error, uint32 line) +{ + if (!ok) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_IDENTITY_TESTS_UNWRAP, error, host_Error_Debug_fmt); +} + +/* txn_identity_tests_derived returns an object's derived id, which unwraps at a line. */ +static uint64 +txn_identity_tests_derived(TxnIdentityKind kind, uint64 parent, const char *name, size_t name_len, uint32 line) +{ + uint64 id; + HostError error; + + txn_identity_tests_ok(txn_identity_derive_object_id(kind, parent, (const uint8 *) name, name_len, &id, &error), + &error, line); + return id; +} + +/* txn_identity_tests_eq reports whether a key is the concatenation of a prefix and a big-endian number's bytes. */ +static bool +txn_identity_tests_eq(const StdVec *key, const uint8 *prefix, size_t prefix_len, uint64 number, size_t number_len) +{ + if (key->len != prefix_len + number_len || memcmp(key->ptr, prefix, prefix_len) != 0) + return false; + for (size_t k = 0; k < number_len; k++) + if (((const uint8 *) key->ptr)[prefix_len + k] != (uint8) (number >> (8 * (number_len - 1 - k)))) + return false; + return true; +} + +/* txn_identity_tests_starts reports whether a key starts with a prefix's bytes. */ +static bool +txn_identity_tests_starts(const StdVec *key, const StdVec *prefix) +{ + return key->len >= prefix->len && memcmp(key->ptr, prefix->ptr, prefix->len) == 0; +} + +/* + * test_txn_identity_tests_an_objects_id_derives_from_its_kind_parent_and_name: + * §II.12's derivation, the same kind, parent, and name give the same id, with + * the user bit set; another name or parent gives another; a taken id moves + * to the next free one. + */ +void +test_txn_identity_tests_an_objects_id_derives_from_its_kind_parent_and_name(void) +{ + uint64 schema = txn_identity_tests_derived(TXN_IDENTITY_KIND_SCHEMA, 0, "public", 6, __LINE__); + uint64 table = txn_identity_tests_derived(TXN_IDENTITY_KIND_RELATION, schema, "t", 1, __LINE__); + + STD_ASSERT_MSG(__LINE__, 9, (schema & TXN_IDENTITY_OBJECT_ID_BIT) == TXN_IDENTITY_OBJECT_ID_BIT, + "schema & OBJECT_ID_BIT == OBJECT_ID_BIT"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_derived(TXN_IDENTITY_KIND_SCHEMA, 0, "public", 6, __LINE__) == schema, + "the same derivation gives the same id"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_derived(TXN_IDENTITY_KIND_RELATION, schema, "u", 1, __LINE__) != + table, "another name gives another id"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_derived(TXN_IDENTITY_KIND_RELATION, table, "t", 1, __LINE__) != + table, "another parent gives another id"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_next_id(table) == ((table + 1) | TXN_IDENTITY_OBJECT_ID_BIT), + "next_id(table) == (table + 1) | OBJECT_ID_BIT"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_next_id(PG_UINT64_MAX) == TXN_IDENTITY_OBJECT_ID_BIT, + "next_id(u64::MAX) == OBJECT_ID_BIT"); +} + +/* txn_identity_tests_column returns a column's derived id under parent 7, by its name and category. */ +static uint64 +txn_identity_tests_column(uint8 category) +{ + StdVec name; + uint64 id; + + txn_identity_column_name((const uint8 *) "a", 1, category, &name); + id = txn_identity_tests_derived(TXN_IDENTITY_KIND_COLUMN, 7, name.ptr, name.len, __LINE__); + std_vec_free_Checked(&name, 1, 1); + return id; +} + +/* + * test_txn_identity_tests_a_columns_id_holds_its_types_category: a column's + * name for its id holds its type's category, so the same name with another + * category is another column, and the same name and category the same + * column. + */ +void +test_txn_identity_tests_a_columns_id_holds_its_types_category(void) +{ + uint64 numeric = txn_identity_tests_column('N'); + uint64 text = txn_identity_tests_column('S'); + + STD_ASSERT_MSG(__LINE__, 9, numeric !=text, "int != text"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_column('N') == numeric, "the same name and category, the same id"); +} + +/* + * test_txn_identity_tests_a_pairs_keys_find_it_from_either_side: the two key + * forms of a pair, and a table's columns' prefix, which every column key of + * the table starts with. + */ +void +test_txn_identity_tests_a_pairs_keys_find_it_from_either_side(void) +{ + static const uint8 by_id_prefix[] = {1, 2}; + static const uint8 by_oid_prefix[] = {2, 2}; + static const uint8 column_prefix[] = {2, 3, 0, 0, 0, 0, 0, 0, 0, 9}; + StdVec key; + StdVec column; + StdVec prefix; + + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, 5, &key); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_eq(&key, by_id_prefix, 2, 5, 8), + "by_id(Kind::Relation, 5) == [1, 2] ++ 5u64.to_be_bytes()"); + std_vec_free_Checked(&key, 1, 1); + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, 16384, &key); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_eq(&key, by_oid_prefix, 2, 16384, 4), + "by_oid(Kind::Relation, 16384) == [2, 2] ++ 16384u32.to_be_bytes()"); + std_vec_free_Checked(&key, 1, 1); + txn_identity_column_by_number(9, 3, &column); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_eq(&column, column_prefix, sizeof(column_prefix), 3, 2), + "column_by_number(9, 3) == [2, 3] ++ 9u64.to_be_bytes() ++ 3u16.to_be_bytes()"); + txn_identity_columns(9, false, &prefix); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_starts(&column, &prefix), "column.starts_with(&columns(9, false))"); + std_vec_free_Checked(&prefix, 1, 1); + std_vec_free_Checked(&column, 1, 1); + txn_identity_column_by_id(9, 11, &column); + txn_identity_columns(9, true, &prefix); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_starts(&column, &prefix), + "column_by_id(9, 11).starts_with(&columns(9, true))"); + std_vec_free_Checked(&prefix, 1, 1); + std_vec_free_Checked(&column, 1, 1); +} + +/* txn_identity_tests_writing_t is writing's (Writer, Roomed): the running transaction's writer and its entry's room. */ +typedef struct txn_identity_tests_writing_t +{ + TxnMemtableVersionWriter writer; + TxnStatusRoomed roomed; +} txn_identity_tests_writing_t; + +/* txn_identity_tests_writing returns the running transaction's writer at its current command, with its entry's room taken. */ +static txn_identity_tests_writing_t +txn_identity_tests_writing(HostIoIo *io) +{ + txn_identity_tests_writing_t writing = {0}; + HostXid sub; + HostXid top; + HostError error; + + txn_identity_tests_ok(host_current_writer(HOST_IO_IO_IO_DOORS(io), &sub, &top, &error) && + txn_status_roomed(io, top, &writing.roomed, &error), &error, __LINE__); + writing.writer.xid = sub.f0; + writing.writer.has_top = sub.f0 != top.f0; + writing.writer.top = top.f0; + return writing; +} + +/* txn_identity_tests_committed commits the running transaction as Postgres' commit leaves its entry once the commit's slot is settled: committed at the WAL's end. */ +static void +txn_identity_tests_committed(HostIoIo *io) +{ + HostXid sub; + HostXid top; + TxnStatusEntry entry; + TxnStatusOutcome to = {.tag = TXN_STATUS_OUTCOME_COMMITTED}; + bool has; + bool settled = false; + HostError error; + + txn_identity_tests_ok(host_current_writer(HOST_IO_IO_IO_DOORS(io), &sub, &top, &error), &error, __LINE__); + simc_end(true); + txn_identity_tests_ok(txn_status_lookup(io, top, &has, &entry, &error), &error, __LINE__); + STD_ASSERT_MSG(__LINE__, 0, has, "status::lookup(io, top).unwrap().unwrap()"); + txn_identity_tests_ok(host_fold_point(HOST_IO_IO_IO_DOORS(io), &to.u.committed, &error) && + txn_status_settled(io, top, entry.outcome, to, &settled, &error), &error, __LINE__); + STD_ASSERT_MSG(__LINE__, 9, settled, "status::settled(io, top, entry.outcome, Committed(end)).unwrap()"); +} + +/* txn_identity_tests_repository makes a test's database's repository and returns its number. */ +static uint32 +txn_identity_tests_repository(HostIoIo *io, uint32 number) +{ + RepoRepo repo; + ChunksErrorError failed; + + if (!repo_Repo_create(io, number, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, TXN_IDENTITY_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + repo_Repo_drop(&repo); + return number; +} + +/* txn_identity_tests_opened_t is opened's database: the repository opened, a reader, and its number. */ +typedef struct txn_identity_tests_opened_t +{ + RepoRepo repo; + HostSharedReader reading; + uint32 repository; +} txn_identity_tests_opened_t; + +/* txn_identity_tests_opened opens a test's database's repository and a reader, the identity tree as its readers reach it. */ +static txn_identity_tests_opened_t +txn_identity_tests_opened(HostIoIo *io, uint32 repository) +{ + txn_identity_tests_opened_t opened = {.repository = repository}; + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_open(io, repository, HOST_PG_DEFAULT, &opened.repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, TXN_IDENTITY_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + txn_identity_tests_ok(host_shared_read(io, &opened.reading, &error), &error, __LINE__); + return opened; +} + +/* txn_identity_tests_opened_database returns an opened repository as the identity tree's database. */ +static TxnIdentityDatabase +txn_identity_tests_opened_database(const txn_identity_tests_opened_t *opened) +{ + TxnIdentityDatabase database = {&opened->repo, &opened->reading, opened->repository}; + + return database; +} + +/* txn_identity_tests_opened_drop frees what opened returned. */ +static void +txn_identity_tests_opened_drop(txn_identity_tests_opened_t *opened) +{ + host_shared_Reader_drop(&opened->reading); + repo_Repo_drop(&opened->repo); +} + +/* txn_identity_tests_keyed is a table's keyed closure: an id's pair by id, as a relation. */ +static void +txn_identity_tests_keyed(const void *captures, uint64 id, StdVec *result) +{ + (void) captures; + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, id, result); +} + +/* txn_identity_tests_table records a table's pairs under SCHEMA in the running transaction and returns its id. */ +static uint64 +txn_identity_tests_table(HostIoIo *io, uint32 repository, const char *name, uint32 oid) +{ + txn_identity_tests_writing_t written = txn_identity_tests_writing(io); + txn_identity_tests_opened_t opened = txn_identity_tests_opened(io, repository); + HostIoWaits waits = host_io_Waits_testing(); + StdVec numbered; + uint64 id; + HostError error; + + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, oid, &numbered); + txn_identity_tests_ok(txn_identity_derived(io, &waits, txn_identity_tests_opened_database(&opened), written.writer, + written.roomed, TXN_IDENTITY_KIND_RELATION, TXN_IDENTITY_TESTS_SCHEMA, + (const uint8 *) name, strlen(name), numbered.ptr, numbered.len, oid, + txn_identity_tests_keyed, NULL, NULL, &id, &error), &error, __LINE__); + std_vec_free_Checked(&numbered, 1, 1); + txn_identity_tests_opened_drop(&opened); + return id; +} + +/* txn_identity_tests_read reads a key of a database's identity tree for the running transaction, answering whether it holds one, into value when given. */ +static bool +txn_identity_tests_read(HostIoIo *io, uint32 repository, const StdVec *key, StdVec *value) +{ + txn_identity_tests_opened_t opened = txn_identity_tests_opened(io, repository); + HostIoWaits waits = host_io_Waits_testing(); + StdVec read; + bool has; + HostError error; + + txn_identity_tests_ok(txn_identity_looked_up(io, &waits, txn_identity_tests_opened_database(&opened), key->ptr, + key->len, &has, &read, &error), &error, __LINE__); + txn_identity_tests_opened_drop(&opened); + if (has && value != NULL) + *value = read; + else if (has) + std_vec_free_Checked(&read, 1, 1); + return has; +} + +/* txn_identity_tests_read_is reports whether a key of a database's identity tree reads as some bytes. */ +static bool +txn_identity_tests_read_is(HostIoIo *io, uint32 repository, const StdVec *key, const uint8 *want, size_t want_len) +{ + StdVec value; + bool is; + + if (!txn_identity_tests_read(io, repository, key, &value)) + return false; + is = value.len == want_len && memcmp(value.ptr, want, want_len) == 0; + std_vec_free_Checked(&value, 1, 1); + return is; +} + +/* txn_identity_tests_id_bytes returns an id's big-endian bytes. */ +static void +txn_identity_tests_id_bytes(uint64 id, uint8 *bytes) +{ + for (int k = 0; k < 8; k++) + bytes[k] = (uint8) (id >> (8 * (7 - k))); +} + +/* TXN_IDENTITY_TESTS_NUMBER is the pair by id's value of OID 16384, its varint. */ +static const uint8 TXN_IDENTITY_TESTS_NUMBER[] = {0x80, 0x80, 0x01}; + +/* txn_identity_tests_pairs_read reports whether a table's two pairs read as its OID's and its id's. */ +static bool +txn_identity_tests_pairs_read(HostIoIo *io, uint32 repository, uint32 oid, uint64 id) +{ + StdVec by_oid; + StdVec by_id; + uint8 bytes[8]; + bool read; + + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, oid, &by_oid); + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, id, &by_id); + txn_identity_tests_id_bytes(id, bytes); + read = txn_identity_tests_read_is(io, repository, &by_oid, bytes, sizeof(bytes)) && + txn_identity_tests_read_is(io, repository, &by_id, TXN_IDENTITY_TESTS_NUMBER, + sizeof(TXN_IDENTITY_TESTS_NUMBER)); + std_vec_free_Checked(&by_id, 1, 1); + std_vec_free_Checked(&by_oid, 1, 1); + return read; +} + +/* txn_identity_tests_pairs_gone reports whether neither of a table's two pairs reads. */ +static bool +txn_identity_tests_pairs_gone(HostIoIo *io, uint32 repository, uint32 oid, uint64 id) +{ + StdVec by_oid; + StdVec by_id; + bool gone; + + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, oid, &by_oid); + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, id, &by_id); + gone = !txn_identity_tests_read(io, repository, &by_oid, NULL) && + !txn_identity_tests_read(io, repository, &by_id, NULL); + std_vec_free_Checked(&by_id, 1, 1); + std_vec_free_Checked(&by_oid, 1, 1); + return gone; +} + +/* txn_identity_tests_logged_count returns how many database change records the simulation logged. */ +static size_t +txn_identity_tests_logged_count(void) +{ + StdVec logged = simc_logged(); + size_t count = 0; + + for (size_t k = 0; k < logged.len; k++) + { + simc_TupleU8VecU8Lsn_t *one = &STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, k); + HostWalKind kind; + + count += host_WalKind_of(one->f0, &kind) && kind == HOST_WAL_KIND_DATABASE_CHANGE; + std_vec_free_Checked(&one->f1, 1, 1); + } + std_vec_free_Checked(&logged, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + return count; +} + +/* txn_identity_tests_versions returns how many identity versions the delta holds at a key. */ +static size_t +txn_identity_tests_versions(HostIoIo *io, uint32 repository, const StdVec *key) +{ + HostSharedHeld deltas; + HostDeltaHold hold; + StdVec versions; + size_t count; + HostError error; + + txn_identity_tests_ok(txn_memtable_branch_deltas(io, repository, HOST_PG_DEFAULT, &deltas, &error) && + host_delta_hold(io, &hold, &error) && + txn_memtable_branch_identity_versions(io, &hold, host_shared_Held_record(&deltas), key->ptr, + key->len, &versions, &error), &error, __LINE__); + count = versions.len; + for (size_t k = 0; k < versions.len; k++) + txn_memtable_version_Version_drop(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, k)); + std_vec_free_Checked(&versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&deltas); + return count; +} + +/* txn_identity_tests_dropped drops a repository's branch state, which unwraps at a line. */ +static void +txn_identity_tests_dropped(HostIoIo *io, uint32 repository, uint32 line) +{ + HostError error; + + txn_identity_tests_ok(txn_memtable_branch_dropped(io, repository, &error), &error, line); +} + +/* + * test_txn_identity_tests_a_tables_pairs_read_back_by_its_oid_and_its_id: + * PLAN item 10's pairs, a table's derivation records its pair by OID and its + * pair by id, which read back in its transaction and, once it commits, in + * the next; the id is the derivation's, and recording the table again finds + * its id and pushes nothing. + */ +void +test_txn_identity_tests_a_tables_pairs_read_back_by_its_oid_and_its_id(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 repository; + uint64 id; + size_t logged; + StdVec by_id; + HostError error; + + simc_end(false); + repository = txn_identity_tests_repository(io, 9470); + id = txn_identity_tests_table(io, repository, "t", 16384); + STD_ASSERT_MSG(__LINE__, 9, id == txn_identity_tests_derived(TXN_IDENTITY_KIND_RELATION, TXN_IDENTITY_TESTS_SCHEMA, + "t", 1, __LINE__), + "id == derive_object_id(Kind::Relation, SCHEMA, b\"t\").unwrap()"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_pairs_read(io, repository, 16384, id), + "the table's pairs read back in its transaction"); + logged = txn_identity_tests_logged_count(); + simc_end(true); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_table(io, repository, "t", 16384) == id, + "recording the table again finds its id"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_logged_count() == logged, "recording it again pushed nothing"); + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, id, &by_id); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_read_is(io, repository, &by_id, TXN_IDENTITY_TESTS_NUMBER, + sizeof(TXN_IDENTITY_TESTS_NUMBER)), + "read(io, repository, &by_id(Kind::Relation, id)) == Some(vec![0x80, 0x80, 0x01])"); + std_vec_free_Checked(&by_id, 1, 1); + simc_end(true); + txn_identity_tests_dropped(io, repository, __LINE__); + txn_identity_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * test_txn_identity_tests_an_aborted_transactions_pairs_go_at_the_take_back: + * 2g's take-back for the identity tree, an aborted transaction's pairs go at + * its abort, so its table has none. + */ +void +test_txn_identity_tests_an_aborted_transactions_pairs_go_at_the_take_back(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 repository; + uint64 id; + StdVec by_id; + HostError error; + + simc_end(false); + repository = txn_identity_tests_repository(io, 9471); + id = txn_identity_tests_table(io, repository, "t", 16384); + simc_end(false); + txn_identity_tests_ok(txn_written_aborted(io, 0, &error), &error, __LINE__); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_pairs_gone(io, repository, 16384, id), + "an aborted transaction's pairs went at its abort"); + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, id, &by_id); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_versions(io, repository, &by_id) == 0, + "branch::identity_versions(...).unwrap().is_empty()"); + std_vec_free_Checked(&by_id, 1, 1); + txn_identity_tests_dropped(io, repository, __LINE__); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * test_txn_identity_tests_a_taken_id_moves_on_and_a_removed_pair_frees_it: + * I14 in one process, a derivation whose id another object's live pair holds + * takes the next free id, and once that object's pairs are removed, an + * object made again under its name derives its id again. + */ +void +test_txn_identity_tests_a_taken_id_moves_on_and_a_removed_pair_frees_it(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 repository; + uint64 first; + txn_identity_tests_writing_t written; + HostIoWaits waits = host_io_Waits_testing(); + StdVec keys; + StdVec key; + HostError error; + + simc_end(false); + repository = txn_identity_tests_repository(io, 9472); + first = txn_identity_tests_table(io, repository, "t", 16384); + simc_end(true); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_table(io, repository, "t", 16390) == txn_identity_next_id(first), + "a taken id moves on to the next free one"); + simc_end(true); + written = txn_identity_tests_writing(io); + std_vec_new(&keys, sizeof(StdVec), _Alignof(StdVec)); + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, 16384, &key); + std_vec_push_Checked(&keys, &key, sizeof(key), _Alignof(StdVec)); + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, first, &key); + std_vec_push_Checked(&keys, &key, sizeof(key), _Alignof(StdVec)); + txn_identity_tests_ok(txn_identity_removed(io, &waits, repository, HOST_PG_DEFAULT, written.writer, written.roomed, + &keys, &error), &error, __LINE__); + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, 16384, &key); + STD_ASSERT_MSG(__LINE__, 9, !txn_identity_tests_read(io, repository, &key, NULL), + "read(io, repository, &by_oid(Kind::Relation, 16384)) == None"); + std_vec_free_Checked(&key, 1, 1); + simc_end(true); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_table(io, repository, "t", 16400) == first, + "a removed pair frees its id"); + simc_end(true); + txn_identity_tests_dropped(io, repository, __LINE__); + txn_identity_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * test_txn_identity_tests_redone_identity_changes_give_the_same_pairs_once: + * I12's case for the identity tree, a database's logged identity changes, + * redone into another database's delta with no catalog, give it the same + * pairs, and a change redone again is held by its LSN and pushes nothing. + */ +void +test_txn_identity_tests_redone_identity_changes_give_the_same_pairs_once(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 primary; + uint32 replica; + uint64 id; + StdVec logged; + StdVec records; + StdVec by_id; + HostError error; + + simc_end(false); + primary = txn_identity_tests_repository(io, 9473); + replica = txn_identity_tests_repository(io, 9474); + id = txn_identity_tests_table(io, primary, "t", 16384); + simc_end(true); + logged = simc_logged(); + std_vec_new(&records, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + for (size_t k = 0; k < logged.len; k++) + { + simc_TupleU8VecU8Lsn_t *one = &STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, k); + TxnRecordDatabaseChange change; + HostWalKind kind; + bool kept = false; + + if (host_WalKind_of(one->f0, &kind) && kind == HOST_WAL_KIND_DATABASE_CHANGE) + { + if (!txn_record_DatabaseChange_decode(one->f1.ptr, one->f1.len, &change, &error)) + host_Error_drop(&error); + else + { + kept = change.repository == primary; + txn_record_DatabaseChange_drop(&change); + } + } + if (kept) + std_vec_push_Checked(&records, one, sizeof(*one), _Alignof(simc_TupleU8VecU8Lsn_t)); + else + std_vec_free_Checked(&one->f1, 1, 1); + } + std_vec_free_Checked(&logged, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + STD_ASSERT_MSG(__LINE__, 9, records.len == 2, "records.len() == 2"); + for (int pass = 0; pass < 2; pass++) + for (size_t k = 0; k < records.len; k++) + { + simc_TupleU8VecU8Lsn_t *one = &STD_VEC_AT(&records, simc_TupleU8VecU8Lsn_t, k); + TxnRecordDatabaseChange change; + + txn_identity_tests_ok(txn_record_DatabaseChange_decode(one->f1.ptr, one->f1.len, &change, &error), &error, + __LINE__); + change.repository = replica; + txn_identity_tests_ok(txn_identity_redo(io, &change, HOST_PG_DEFAULT, one->f2, &error), &error, + __LINE__); + } + for (size_t k = 0; k < records.len; k++) + std_vec_free_Checked(&STD_VEC_AT(&records, simc_TupleU8VecU8Lsn_t, k).f1, 1, 1); + std_vec_free_Checked(&records, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_pairs_read(io, replica, 16384, id), + "the replica's pairs read as the primary's"); + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, id, &by_id); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_versions(io, replica, &by_id) == 1, + "a change redone again pushes nothing"); + std_vec_free_Checked(&by_id, 1, 1); + txn_identity_tests_dropped(io, primary, __LINE__); + txn_identity_tests_dropped(io, replica, __LINE__); + txn_identity_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* txn_identity_tests_refused reports whether a database change record's body is refused. */ +static bool +txn_identity_tests_refused(ChunksRecordRecordWriter *w) +{ + StdVec body = chunks_record_RecordWriter_finish(w); + TxnRecordDatabaseChange change; + HostError error; + bool refused = !txn_record_DatabaseChange_decode(body.ptr, body.len, &change, &error); + + if (refused) + host_Error_drop(&error); + else + txn_record_DatabaseChange_drop(&change); + std_vec_free_Checked(&body, 1, 1); + return refused; +} + +/* + * test_txn_identity_tests_an_identity_change_reads_back_as_written: the + * database change record's form, a subtransaction's writer, a put's value, + * and a removal's lack of one read back as written, and an unknown field + * and a tree none is are refused. + */ +void +test_txn_identity_tests_an_identity_change_reads_back_as_written(void) +{ + TxnMemtableVersionWriter writer = {.xid = 41, .has_top = true, .top = 40, .command = 3}; + ChunksRecordRecordWriter w; + + for (int put = 0; put < 2; put++) + { + TxnRecordDatabaseChange change = {.tree = TXN_RECORD_TREE_IDENTITY, .writer = writer, .repository = 9475, + .has_value = put == 0}; + TxnRecordDatabaseChange read; + StdVec body; + HostError error; + + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, 16384, &change.key); + if (put == 0) + std_slice_to_vec_Checked(&change.value, "v", 1, 1, 1); + txn_record_DatabaseChange_encode(&change, &body); + txn_identity_tests_ok(txn_record_DatabaseChange_decode(body.ptr, body.len, &read, &error), &error, __LINE__); + STD_ASSERT_MSG(__LINE__, 13, txn_record_DatabaseChange_PartialEq_eq(&read, &change), + "DatabaseChange::decode(&change.encode()).unwrap() == change"); + txn_record_DatabaseChange_drop(&read); + txn_record_DatabaseChange_drop(&change); + std_vec_free_Checked(&body, 1, 1); + } + w = chunks_record_RecordWriter_with_capacity(32); + chunks_record_RecordWriter_uint(chunks_record_RecordWriter_uint(chunks_record_RecordWriter_uint( + chunks_record_RecordWriter_uint(&w, 1, 1), 2, 9475), 3, 40), 5, 0); + chunks_record_RecordWriter_uint(chunks_record_RecordWriter_field(&w, 6, (const uint8 *) "k", 1), 8, 1); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_refused(&w), "an unknown field read back"); + w = chunks_record_RecordWriter_with_capacity(32); + chunks_record_RecordWriter_uint(chunks_record_RecordWriter_uint(chunks_record_RecordWriter_uint( + chunks_record_RecordWriter_uint(&w, 1, 2), 2, 9475), 3, 40), 5, 0); + chunks_record_RecordWriter_field(&w, 6, (const uint8 *) "k", 1); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_refused(&w), "a tree none is read back"); +} + +/* txn_identity_tests_gather is a save's gather of the folded pairs, each as the identity tree's pair. */ +static bool +txn_identity_tests_gather(void *captures, HostIoIo *io, const RepoRead *read, StdVec *result, ChunksErrorError *error) +{ + const StdVec *pairs = captures; + + (void) io; + (void) read; + (void) error; + std_vec_new(result, sizeof(RepoGathered), _Alignof(RepoGathered)); + for (size_t k = 0; k < pairs->len; k++) + { + const TxnIdentityPair *pair = &STD_VEC_AT(pairs, TxnIdentityPair, k); + RepoGathered gathered = {.tag = REPO_GATHERED_PAIR, .u.pair = {.has_value = pair->has_f1}}; + + std_slice_to_vec_Checked(&gathered.u.pair.key, pair->f0.ptr, pair->f0.len, 1, 1); + if (pair->has_f1) + std_slice_to_vec_Checked(&gathered.u.pair.value, pair->f1.ptr, pair->f1.len, 1, 1); + std_vec_push_Checked(result, &gathered, sizeof(gathered), _Alignof(RepoGathered)); + } + return true; +} + +/* txn_identity_tests_saved is a save's after-write call, which does nothing. */ +static bool +txn_identity_tests_saved(void *captures, HostIoIo *io, ChunksErrorError *error) +{ + (void) captures; + (void) io; + (void) error; + return true; +} + +/* txn_identity_tests_folded folds a repository's committed pairs at a point and saves them into the tree the manifest names, returning the pairs. */ +static StdVec +txn_identity_tests_folded(HostIoIo *io, uint32 repository, HostLsn point) +{ + StdVec pairs; + txn_identity_tests_opened_t opened; + ChunksErrorError failed; + HostError error; + + txn_identity_tests_ok(txn_identity_folded(io, repository, point, &pairs, &error), &error, __LINE__); + opened = txn_identity_tests_opened(io, repository); + if (!repo_Repo_save(&opened.repo, io, &opened.reading, false, NULL, txn_identity_tests_gather, &pairs, + txn_identity_tests_saved, NULL, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, TXN_IDENTITY_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + txn_identity_tests_opened_drop(&opened); + return pairs; +} + +/* txn_identity_tests_pairs_drop drops folded's pairs. */ +static void +txn_identity_tests_pairs_drop(StdVec *pairs) +{ + for (size_t k = 0; k < pairs->len; k++) + { + TxnIdentityPair *pair = &STD_VEC_AT(pairs, TxnIdentityPair, k); + + std_vec_free_Checked(&pair->f0, 1, 1); + if (pair->has_f1) + std_vec_free_Checked(&pair->f1, 1, 1); + } + std_vec_free_Checked(pairs, sizeof(TxnIdentityPair), _Alignof(TxnIdentityPair)); +} + +/* txn_identity_tests_point returns the fold point the running fold position lowers to. */ +static HostLsn +txn_identity_tests_point(HostIoIo *io) +{ + HostLsn position; + HostLsn point = 0; + HostError error; + + txn_identity_tests_ok(host_fold_point(HOST_IO_IO_IO_DOORS(io), &position, &error) && + txn_status_lowered(io, position, &point, &error), &error, __LINE__); + return point; +} + +/* + * test_txn_identity_tests_a_saves_fold_keeps_its_committed_pairs_and_leaves_running_ones: + * §III.6's fold of the identity tree at a save, a save folds the pairs whose + * writers committed at or below its point into the tree the manifest names, + * a running writer's pair staying in the delta; with the delta's versions + * gone, the folded pairs read from the tree alone, and the running writer's + * pair is gone with them. + */ +void +test_txn_identity_tests_a_saves_fold_keeps_its_committed_pairs_and_leaves_running_ones(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 repository; + uint64 id; + uint64 running; + HostLsn point; + StdVec pairs; + HostError error; + + simc_end(false); + repository = txn_identity_tests_repository(io, 9477); + id = txn_identity_tests_table(io, repository, "t", 16384); + txn_identity_tests_committed(io); + point = txn_identity_tests_point(io); + running = txn_identity_tests_table(io, repository, "u", 16385); + pairs = txn_identity_tests_folded(io, repository, point); + STD_ASSERT_MSG(__LINE__, 9, pairs.len == 2, "pairs.len() == 2"); + txn_identity_tests_pairs_drop(&pairs); + simc_end(false); + txn_identity_tests_ok(txn_written_aborted(io, 0, &error), &error, __LINE__); + txn_identity_tests_dropped(io, repository, __LINE__); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_pairs_read(io, repository, 16384, id), + "the folded pairs read from the tree alone"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_pairs_gone(io, repository, 16385, running), + "the running writer's pair is gone"); + txn_identity_tests_dropped(io, repository, __LINE__); + txn_identity_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* txn_identity_tests_kept returns how many identity versions the delta holds at a table's two pairs. */ +static size_t +txn_identity_tests_kept(HostIoIo *io, uint32 repository, uint32 oid, uint64 id) +{ + StdVec key; + size_t kept; + + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, oid, &key); + kept = txn_identity_tests_versions(io, repository, &key); + std_vec_free_Checked(&key, 1, 1); + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, id, &key); + kept += txn_identity_tests_versions(io, repository, &key); + std_vec_free_Checked(&key, 1, 1); + return kept; +} + +/* txn_identity_tests_counted returns an xid's entry's count, 0 for none. */ +static uint64 +txn_identity_tests_counted(HostIoIo *io, HostXid top) +{ + TxnStatusEntry entry; + bool has; + HostError error; + + txn_identity_tests_ok(txn_status_lookup(io, top, &has, &entry, &error), &error, __LINE__); + return has ? entry.count : 0; +} + +/* + * test_txn_identity_tests_a_saves_drop_takes_its_folded_pairs_out: bugs row + * 164, a save's drop takes the identity versions it folded out of the delta + * and gives their counts back at their writer's entry, the pairs then + * reading from the manifest's tree alone. + */ +void +test_txn_identity_tests_a_saves_drop_takes_its_folded_pairs_out(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 repository; + uint64 id; + HostXid sub; + HostXid top; + HostLsn point; + StdVec pairs; + HostError error; + + simc_end(false); + repository = txn_identity_tests_repository(io, 9479); + id = txn_identity_tests_table(io, repository, "t", 16384); + txn_identity_tests_ok(host_current_writer(HOST_IO_IO_IO_DOORS(io), &sub, &top, &error), &error, __LINE__); + txn_identity_tests_committed(io); + point = txn_identity_tests_point(io); + pairs = txn_identity_tests_folded(io, repository, point); + txn_identity_tests_pairs_drop(&pairs); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_kept(io, repository, 16384, id) == 2 && + txn_identity_tests_counted(io, top) == 2, "(kept(io), counted(io)) == (2, 2)"); + { + bool took; + + txn_identity_tests_ok(txn_identity_dropped(io, repository, point, &took, &error), &error, __LINE__); + } + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_kept(io, repository, 16384, id) == 0 && + txn_identity_tests_counted(io, top) == 0, "(kept(io), counted(io)) == (0, 0)"); + STD_ASSERT_MSG(__LINE__, 9, txn_identity_tests_pairs_read(io, repository, 16384, id), + "the pairs read from the manifest's tree alone"); + txn_identity_tests_dropped(io, repository, __LINE__); + txn_identity_tests_ok(txn_written_ended(io, &error), &error, __LINE__); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +#endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/insert.c b/src/doltgres/txn/c/insert.c index 91e7516530b78..3b7d74625821b 100644 --- a/src/doltgres/txn/c/insert.c +++ b/src/doltgres/txn/c/insert.c @@ -788,6 +788,17 @@ txn_insert_Seen_drop(TxnInsertSeen *self) self->has_current = self->has_below = false; } +/* txn_insert_Seen_read is a take's read of a block whose entry it saw outside the window, with the base generation read before it. */ +static TxnMemtableMapRead +txn_insert_Seen_read(const TxnInsertSeen *self, const TxnInsertBase *base) +{ + return (TxnMemtableMapRead) + { + .has_entry = self->has_current, .entry = self->has_current ? self->current.ptr : NULL, + .entry_len = self->has_current ? self->current.len : 0, .generation = base->generation + }; +} + /* txn_insert_seen_frame_t is seen's state across its pad: what it read, while live. */ typedef struct txn_insert_seen_frame_t { @@ -802,7 +813,7 @@ typedef struct txn_insert_seen_frame_t bool ok; } txn_insert_seen_frame_t; -/* txn_insert_seen_body reads the block's newest entry, the delta's else the base's, and decodes it. */ +/* txn_insert_seen_body reads the block's newest entry, the memtable's, else its spills', else the base's, and decodes it. */ static void txn_insert_seen_body(void *frame_) { @@ -813,15 +824,28 @@ txn_insert_seen_body(void *frame_) ChunksErrorError failed; uint8 key[4]; bool some; + bool spilled; - if (!txn_memtable_map_newest(frame->io, frame->record, TXN_MEMTABLE_MAP_MAP_FREE, frame->block, &some, &newest, - frame->error)) + if (!txn_memtable_map_in_memtable(frame->io, frame->record, TXN_MEMTABLE_MAP_MAP_FREE, frame->block, &some, + &newest, frame->error)) return; if (some) { result->has_current = newest.has_value; result->current = newest.value; } + else if (!txn_memtable_map_spilled_newest(frame->io, frame->record, frame->block, &spilled, &newest, frame->error)) + return; + else if (spilled) + { + result->has_below = newest.has_value; + result->below = newest.value; + if (result->has_below) + { + std_slice_to_vec_Checked(&result->current, result->below.ptr, result->below.len, 1, 1); + result->has_current = true; + } + } else if (frame->base->has_tree) { TreesStoreTree tree = frame->base->tree; @@ -860,7 +884,8 @@ txn_insert_seen_pad(void *frame_) * txn_insert_seen reads a block's entry outside any window, the delta's * newest else the base's, and decodes it there, so a corrupt entry raises * XX001 outside the critical section; it returns the bytes a take compares - * under its lock, and the base's entry the take falls back to. + * under its lock, and the entry below the memtable the take falls back to: + * the spills' newest, else the base's. */ static bool txn_insert_seen(HostIoIo *io, TreesStoreNodeStore store, HostSharedRecord record, const TxnInsertBase *base, @@ -1473,11 +1498,9 @@ txn_insert_ended(const TxnInsertInsert *row, const TxnInsertReplaces *replaces, return true; } -/* txn_insert_TakeStep_t is a take's step's captures: what it compares and the entry it reads. */ +/* txn_insert_TakeStep_t is a take's step's captures: the entry it reads. */ typedef struct txn_insert_TakeStep_t { - bool has_expected; - const StdVec *expected; HostSharedRecord record; const TxnInsertBase *base; const TxnInsertRoom *room; @@ -1486,18 +1509,6 @@ typedef struct txn_insert_TakeStep_t uint64 tid; } txn_insert_TakeStep_t; -/* txn_insert_unmoved reports whether a block's entry and the base are as they were read outside the window. */ -static bool -txn_insert_unmoved(const txn_insert_TakeStep_t *step, bool has_current, const uint8 *current, size_t current_len) -{ - if (has_current != step->has_expected) - return false; - if (has_current && (current_len != step->expected->len || - (current_len != 0 && memcmp(current, step->expected->ptr, current_len) != 0))) - return false; - return txn_state_generation(step->record) == step->base->generation; -} - /* * txn_insert_filled is the step of an update whose new row found no room on * its replaced row's block: the block's full flag set, as heap_update sets @@ -1525,10 +1536,9 @@ txn_insert_filled(void *captures, bool has_current, const uint8 *current, size_t /* * txn_insert_take_block is a take's step on a block other than the fresh - * page, pure over its input: Retry when the entry or the base moved since - * they were read outside the window, Refuse when the entry has no room for - * the row, and otherwise the entry with its lowest free offset and the row's - * bytes taken, written into out, or its removal when it was the last. + * page, pure over its input: Refuse when the entry has no room for the row, + * and otherwise the entry with its lowest free offset and the row's bytes + * taken, written into out, or its removal when it was the last. */ static TxnMemtableMapStep txn_insert_take_block(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out, @@ -1538,8 +1548,6 @@ txn_insert_take_block(void *captures, bool has_current, const uint8 *current, si TxnFreeSpaceFree entry; uint16 offset; - if (!txn_insert_unmoved(step, has_current, current, current_len)) - return TXN_MEMTABLE_MAP_STEP_RETRY; if (!has_current) return TXN_MEMTABLE_MAP_STEP_REFUSE; if (!txn_insert_parsed(current, current_len, step->block, step->base, txn_state_counter(step->record), @@ -1560,9 +1568,9 @@ txn_insert_take_block(void *captures, bool has_current, const uint8 *current, si * txn_insert_take_fresh is a take's step on the fresh page, the counter's * block, pure over its input: its lowest freed offset, else the counter's * while the page has room and offsets, else the page left for a new one with - * its counter offsets handed to the map; Retry when the entry, the counter, - * or the base moved since they were read. An untouched page has a page's - * bytes free; a touched one with no entry is full. + * its counter offsets handed to the map; Retry when the counter moved since + * it was read. An untouched page has a page's bytes free; a touched one with + * no entry is full. */ static TxnMemtableMapStep txn_insert_take_fresh(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out, @@ -1575,8 +1583,7 @@ txn_insert_take_fresh(void *captures, bool has_current, const uint8 *current, si uint16 freed; uint16 next; - if (!txn_insert_unmoved(step, has_current, current, current_len) || - txn_state_counter(step->record) != step->counter) + if (txn_state_counter(step->record) != step->counter) return TXN_MEMTABLE_MAP_STEP_RETRY; if (has_current) { @@ -1627,7 +1634,11 @@ txn_insert_fresh_moved(void *captures, const TxnInsertGot *got) txn_state_moved(step->record, step->counter, STD_ADD(u64, step->counter >> 16, 1, 953, 65) << 16 | 1); } -/* txn_insert_stored is a stored value's step: its entry written where the map holds none, else Retry. */ +/* + * txn_insert_stored is a stored value's step: its entry written where the map + * holds none, else Retry. Its take has no read to compare: the value id is + * new, so no entry falls back from under it (bugs row 342). + */ static TxnMemtableMapStep txn_insert_stored(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out) { @@ -1637,7 +1648,12 @@ txn_insert_stored(void *captures, bool has_current, const uint8 *current, size_t return TXN_MEMTABLE_MAP_STEP_PUT; } -/* txn_insert_added is a kept value's step, values' count add. */ +/* + * txn_insert_added is a kept value's step, values' count add. Its take falls + * back to the newest entry its caller read whole, which a spill moves + * unchanged, and the values mark its caller rechecks before the window + * covers a write between, so it has no read to compare (bugs row 342). + */ static TxnMemtableMapStep txn_insert_added(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out) { @@ -1806,6 +1822,27 @@ txn_insert_turn_closure_9_logged(txn_insert_turn_closure_9_frame_t *frame, HostI return true; } +/* + * txn_insert_turn_closure_9_spent logs, for a logged turn that took its + * target, the take of a tuple id the turn keeps without its row. + */ +static bool +txn_insert_turn_closure_9_spent(txn_insert_turn_closure_9_frame_t *frame, HostIoIoDoors doors, uint64 tid, + const TxnInsertRoom *room) +{ + txn_insert_turn_frame_t *turn = frame->turn; + TxnInsertWriting *w = turn->w; + TxnRecordSpent spent = {.writer = w->change.writer, .storage = w->change.storage, .tuple = {tid}, + .heap_bytes = room->bytes}; + HostLsn spent_lsn; + + if (!w->logged || turn->target.tag == TXN_INSERT_TARGET_TAKEN) + return true; + turn->spending_live = false; + txn_record_Spent_encode_with(&spent, &turn->spending, &frame->body); + return txn_insert_turn_closure_9_logged(frame, doors, HOST_WAL_KIND_ROW_CHANGE, &spent_lsn); +} + /* txn_insert_turn_closure_9_body reads under the prepare's hold, rechecks under the stripes, and runs the window. */ static void txn_insert_turn_closure_9_body(void *frame_) @@ -1814,7 +1851,7 @@ txn_insert_turn_closure_9_body(void *frame_) txn_insert_turn_frame_t *turn = frame->turn; TxnInsertWriting *w = turn->w; const HostDeltaHold *hold = txn_memtable_table_PreparedState_hold(frame->prepared); - static const char *const pauses[] = {"doltgres-update-ended", "doltgres-insert-logged"}; + static const char *const pauses[] = {"doltgres-update-ended", "doltgres-insert-logged", "doltgres-insert-flagged-moved"}; bool has_recheck[2] = {false, false}; bool has_newest[2] = {false, false}; TxnMemtableVersionHandle newest[2]; @@ -2009,24 +2046,21 @@ txn_insert_turn_closure_9_body(void *frame_) } else { - txn_insert_TakeStep_t step = {.has_expected = turn->seen.has_current, .expected = &turn->seen.current, - .record = turn->record, .base = turn->base, .room = room, .block = turn->block, + txn_insert_TakeStep_t step = {.record = turn->record, .base = turn->base, .room = room, .block = turn->block, .counter = turn->target.tag == TXN_INSERT_TARGET_FRESH ? turn->target.u.fresh : 0}; + TxnMemtableMapRead read = txn_insert_Seen_read(&turn->seen, turn->base); bool fresh = turn->target.tag == TXN_INSERT_TARGET_FRESH; HostRow unclaimed = {0}; - if (!txn_memtable_map_took_Got(io, frame->prepared, 0, writer, false, 0, false, unclaimed, 0, - turn->seen.has_below, turn->seen.below.ptr, turn->seen.below.len, - fresh ? txn_insert_take_fresh : txn_insert_take_block, &step, NULL, - txn_insert_Got_tuple, NULL, NULL, + if (!txn_memtable_map_took_Got(io, frame->prepared, 0, writer, w->has_roomed, w->roomed, false, 0, false, + unclaimed, 0, turn->seen.has_below, turn->seen.below.ptr, + turn->seen.below.len, &read, fresh ? txn_insert_take_fresh : txn_insert_take_block, + &step, NULL, txn_insert_Got_tuple, NULL, NULL, fresh ? txn_insert_fresh_moved : txn_insert_nothing_got, &step, NULL, &taken, &got, frame->error)) return; - if (taken == TXN_MEMTABLE_MAP_TAKEN_TOOK && - !txn_memtable_table_counted(io, frame->prepared, w->has_roomed, w->roomed, frame->error)) - return; } - if (taken == TXN_MEMTABLE_MAP_TAKEN_RETRY || + if (taken == TXN_MEMTABLE_MAP_TAKEN_RETRY || taken == TXN_MEMTABLE_MAP_TAKEN_MOVED || (taken == TXN_MEMTABLE_MAP_TAKEN_TOOK && got.tag == TXN_INSERT_GOT_LEFT)) { if (taken == TXN_MEMTABLE_MAP_TAKEN_TOOK && w->logged && turn->target.tag == TXN_INSERT_TARGET_FRESH) @@ -2049,17 +2083,8 @@ txn_insert_turn_closure_9_body(void *frame_) return; } tid = got.tid; - if ((has_waiting || has_refused) && w->logged && turn->target.tag != TXN_INSERT_TARGET_TAKEN) - { - TxnRecordSpent spent = {.writer = writer, .storage = w->change.storage, .tuple = {tid}, - .heap_bytes = room->bytes}; - HostLsn spent_lsn; - - turn->spending_live = false; - txn_record_Spent_encode_with(&spent, &turn->spending, &frame->body); - if (!txn_insert_turn_closure_9_logged(frame, doors, HOST_WAL_KIND_ROW_CHANGE, &spent_lsn)) - return; - } + if ((has_waiting || has_refused) && !txn_insert_turn_closure_9_spent(frame, doors, tid, room)) + return; if (has_waiting) { txn_insert_turn_closure_9_finish(frame, TXN_INSERT_TURN_WAIT); @@ -2082,17 +2107,15 @@ txn_insert_turn_closure_9_body(void *frame_) } for (size_t at = 0; at < turn->stored; at++) { - if (!txn_memtable_map_finish_Unit(io, frame->prepared, at + turn->first, writer, false, 0, false, NULL, 0, - txn_insert_stored, (void *) &w->entries[at], NULL, txn_insert_nothing, NULL, - NULL, &taken, frame->error)) + if (!txn_memtable_map_finish_Unit(io, frame->prepared, at + turn->first, writer, w->has_roomed, w->roomed, + false, 0, false, NULL, 0, NULL, txn_insert_stored, (void *) &w->entries[at], NULL, + txn_insert_nothing, NULL, NULL, &taken, frame->error)) return; if (taken != TXN_MEMTABLE_MAP_TAKEN_TOOK) { txn_insert_turn_closure_9_finish(frame, TXN_INSERT_TURN_RETRY); return; } - if (!txn_memtable_table_counted(io, frame->prepared, w->has_roomed, w->roomed, frame->error)) - return; } if (w->toast != NULL) { @@ -2109,6 +2132,39 @@ txn_insert_turn_closure_9_body(void *frame_) return; } } + if (turn->has_flagged) + { + txn_insert_TakeStep_t step = {.record = turn->record, .base = turn->base, .room = room, + .block = turn->flagged}; + TxnMemtableMapRead read = txn_insert_Seen_read(&turn->flagged_seen, turn->base); + + if (!txn_memtable_map_finish_Unit(io, frame->prepared, turn->flagging, writer, w->has_roomed, w->roomed, + false, 0, turn->flagged_seen.has_below, turn->flagged_seen.below.ptr, + turn->flagged_seen.below.len, &read, txn_insert_filled, &step, NULL, + txn_insert_nothing, NULL, NULL, &taken, frame->error)) + return; + if (taken == TXN_MEMTABLE_MAP_TAKEN_MOVED) + { + /* + * The block is read again outside the window, where an error is + * an ordinary one, by a turn that keeps this tuple id, so the + * flag it pushes is the one its record names (bugs row 342). + */ + if (!host_pause(HOST_SHARED_KEY_IO_PAUSES(&key_io), "doltgres-insert-flagged-moved", frame->error) || + !txn_insert_turn_closure_9_spent(frame, doors, tid, room)) + return; + txn_insert_turn_closure_9_finish(frame, TXN_INSERT_TURN_RETRY); + w->has_taken = true; + w->taken = tid; + return; + } + if (taken == TXN_MEMTABLE_MAP_TAKEN_RETRY) + { + std_string_from_Checked(&message, "a free space map's entry is corrupt", 35); + *frame->error = host_Error_fork((const uint8 *) "XX001", &message); + return; + } + } if (w->logged && w->decoded && w->has_replacing && !host_wal_new_cid(doors, &w->change.storage, w->replaces.tuple, txn_change_changed_cid(shown, cmin, writer.command), &cid_lsn, frame->error)) @@ -2177,9 +2233,10 @@ txn_insert_turn_closure_9_body(void *frame_) { const TxnInsertKept *read = &STD_VEC_AT(turn->kept, TxnInsertKept, at); - if (!txn_memtable_map_finish_Unit(io, frame->prepared, turn->first + turn->stored + at, writer, has_lsn, lsn, - read->has_f0, read->f0.ptr, read->f0.len, txn_insert_added, NULL, NULL, - txn_insert_nothing, NULL, NULL, &taken, frame->error)) + if (!txn_memtable_map_finish_Unit(io, frame->prepared, turn->first + turn->stored + at, writer, + w->has_roomed, w->roomed, has_lsn, lsn, read->has_f0, read->f0.ptr, + read->f0.len, NULL, txn_insert_added, NULL, NULL, txn_insert_nothing, NULL, NULL, + &taken, frame->error)) return; if (taken != TXN_MEMTABLE_MAP_TAKEN_TOOK) { @@ -2187,8 +2244,6 @@ txn_insert_turn_closure_9_body(void *frame_) *frame->error = host_Error_fork((const uint8 *) "XX001", &message); return; } - if (!txn_memtable_table_counted(io, frame->prepared, w->has_roomed, w->roomed, frame->error)) - return; } if (turn->marks_live && turn->marks.len > 0) { @@ -2196,9 +2251,8 @@ txn_insert_turn_closure_9_body(void *frame_) StdVec marks = turn->marks; std_vec_new(&turn->marks, TXN_INSERT_ID); - if (!txn_memtable_map_named(io, frame->prepared, turn->marking, writer, has_lsn, lsn, row, &marks, - frame->error) || - !txn_memtable_table_counted(io, frame->prepared, w->has_roomed, w->roomed, frame->error)) + if (!txn_memtable_map_named(io, frame->prepared, turn->marking, writer, w->has_roomed, w->roomed, has_lsn, + lsn, row, &marks, frame->error)) return; } if (!host_pause(HOST_SHARED_KEY_IO_PAUSES(&key_io), "doltgres-insert-logged", frame->error)) @@ -2241,23 +2295,6 @@ txn_insert_turn_closure_9_body(void *frame_) if (!done || !txn_memtable_tuples_finish(io, frame->prepared, row, frame->error)) return; } - if (turn->has_flagged) - { - txn_insert_TakeStep_t step = {.record = turn->record, .base = turn->base, .room = room, - .block = turn->flagged}; - - if (!txn_memtable_map_finish_Unit(io, frame->prepared, turn->flagging, writer, false, 0, - turn->flagged_seen.has_below, turn->flagged_seen.below.ptr, - turn->flagged_seen.below.len, txn_insert_filled, &step, NULL, - txn_insert_nothing, NULL, NULL, &taken, frame->error)) - return; - if (taken == TXN_MEMTABLE_MAP_TAKEN_RETRY) - { - std_string_from_Checked(&message, "a free space map's entry is corrupt", 35); - *frame->error = host_Error_fork((const uint8 *) "XX001", &message); - return; - } - } txn_insert_turn_closure_9_finish(frame, TXN_INSERT_TURN_TOOK); frame->result->u.took = tid; } @@ -2313,7 +2350,7 @@ txn_insert_turn_take(HostSharedRecord state, TxnMemtableMapMap map, uint32 key, */ static bool txn_insert_turn_touched(HostIoIo *io, uint64 top, const HostStorage *storage, TxnWrittenOf of, const uint8 *key, - size_t key_len, size_t need, bool unique, HostStorage over[3], TxnWrittenOf overs[3], + size_t key_len, size_t need, bool unique, HostStorage over[4], TxnWrittenOf overs[4], size_t *over_len, HostError *error) { uint64 touched; @@ -2341,8 +2378,8 @@ txn_insert_turn_body(void *frame_) const TxnMemtablePut *put; size_t free_most = txn_free_space_most_len(w->room->most); uint64 top = change->writer.has_top ? change->writer.top : change->writer.xid; - HostStorage over[3]; - TxnWrittenOf overs[3]; + HostStorage over[4]; + TxnWrittenOf overs[4]; size_t over_len = 0; TxnMemtableMapTake take; txn_TupleRefSliceU8Usize_t row; @@ -2409,9 +2446,8 @@ txn_insert_turn_body(void *frame_) &frame->flagged_seen, frame->error)) return; frame->has_flagged = true; - if (frame->flagged_seen.has_current) - std_vec_free_Checked(&frame->flagged_seen.current, 1, 1); - frame->flagged_seen.has_current = false; + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-insert-flagged-read", frame->error)) + return; } if (txn_record_RowChange_put(change, &put)) std_slice_to_vec_Checked(&frame->marks, put->values.ptr, put->values.len, TXN_INSERT_ID); @@ -2484,6 +2520,18 @@ txn_insert_turn_body(void *frame_) return; } } + for (size_t i = 0; i < frame->takes.len; i++) + { + const TxnMemtableMapTake *take = &STD_VEC_AT(&frame->takes, TxnMemtableMapTake, i); + uint8 block[4]; + + if (take->map != TXN_MEMTABLE_MAP_MAP_FREE) + continue; + std_u32_to_be_bytes(take->key, block); + if (!txn_insert_turn_touched(frame->io, top, &change->storage, TXN_WRITTEN_OF_FREE, block, sizeof(block), + take->most, true, over, overs, &over_len, frame->error)) + return; + } for (size_t i = 0; i < over_len; i++) { bool spilled; @@ -3097,6 +3145,7 @@ typedef struct txn_insert_jump_closure_0_captures_t { TxnMemtableVersionWriter writer; const TxnInsertSeen *seen; + const TxnInsertBase *base; TxnMemtableMapStep (*step) (void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out); txn_insert_JumpStep_t *captures; @@ -3104,20 +3153,18 @@ typedef struct txn_insert_jump_closure_0_captures_t TxnStatusRoomed roomed; } txn_insert_jump_closure_0_captures_t; -/* txn_insert_jump_closure_0 is a jump's prepare's closure: the counter block's take, counted when it took. */ +/* txn_insert_jump_closure_0 is a jump's prepare's closure: the counter block's take. */ static bool txn_insert_jump_closure_0(void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, TxnMemtableMapTaken *result, HostError *error) { txn_insert_jump_closure_0_captures_t *inside = captures; + TxnMemtableMapRead read = txn_insert_Seen_read(inside->seen, inside->base); - if (!txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, inside->writer, false, 0, - inside->seen->has_below, inside->seen->below.ptr, inside->seen->below.len, - inside->step, inside->captures, NULL, txn_insert_jump_moved, inside->captures, - NULL, result, error)) - return false; - return *result != TXN_MEMTABLE_MAP_TAKEN_TOOK || - txn_memtable_table_counted(HOST_IO_IO_TAKES_SHORT(io), prepared, inside->has_roomed, inside->roomed, error); + return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, inside->writer, inside->has_roomed, + inside->roomed, false, 0, inside->seen->has_below, inside->seen->below.ptr, + inside->seen->below.len, &read, inside->step, inside->captures, NULL, + txn_insert_jump_moved, inside->captures, NULL, result, error); } /* txn_insert_jump_frame_t is jump's state across its pad: the counter block's entries it read. */ @@ -3146,20 +3193,42 @@ txn_insert_jump_body(void *frame_) uint32 block = (uint32) (frame->step_captures.counter >> 16); TxnMemtableVersionWriter none = TXN_MEMTABLE_VERSION_NO_TRANSACTION; txn_insert_jump_closure_0_captures_t inside = {.writer = frame->writer, .seen = &frame->seen, - .step = frame->step, .captures = &frame->step_captures}; - TxnMemtableMapTake take; + .base = &frame->base, .step = frame->step, .captures = &frame->step_captures}; + TxnMemtableMapTake take = txn_insert_turn_take(record, TXN_MEMTABLE_MAP_MAP_FREE, block, + txn_free_space_most_len(frame->step_captures.room->most)); + HostStorage storage; + TxnStatePersistence persistence; - if (!txn_insert_read_base(frame->io, frame->held, &frame->base, frame->error) || - !txn_insert_seen(frame->io, frame->store, record, &frame->base, frame->step_captures.room, block, - &frame->seen, frame->error)) - return; - take = txn_insert_turn_take(record, TXN_MEMTABLE_MAP_MAP_FREE, block, - txn_free_space_most_len(frame->step_captures.room->most)); + if (!txn_memtable_version_Writer_PartialEq_eq(&frame->writer, &none) && + txn_state_described(record, &storage, &persistence)) + { + uint64 top = frame->writer.has_top ? frame->writer.top : frame->writer.xid; + uint64 bytes; + bool spilled; + + if (!txn_spill_replayed(frame->io, &storage, TXN_WRITTEN_OF_FREE, top, take.most + 4, &bytes, frame->error)) + return; + if (bytes >= txn_spill_threshold() && + !txn_spill_spilled(frame->io, &storage, TXN_WRITTEN_OF_FREE, top, &spilled, frame->error)) + return; + } if (!txn_memtable_version_Writer_PartialEq_eq(&frame->writer, &none) && !txn_status_redo_roomed(frame->io, &frame->writer, &inside.has_roomed, &inside.roomed, frame->error)) return; - frame->ok = txn_memtable_map_prepare_Taken(frame->io, &take, 1, txn_insert_jump_closure_0, &inside, NULL, - frame->result, frame->error); + for (;;) + { + if (!txn_insert_read_base(frame->io, frame->held, &frame->base, frame->error) || + !txn_insert_seen(frame->io, frame->store, record, &frame->base, frame->step_captures.room, block, + &frame->seen, frame->error) || + !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-insert-jump-seen", frame->error) || + !txn_memtable_map_prepare_Taken(frame->io, &take, 1, txn_insert_jump_closure_0, &inside, NULL, + frame->result, frame->error)) + return; + if (*frame->result != TXN_MEMTABLE_MAP_TAKEN_MOVED) + break; + txn_insert_Seen_drop(&frame->seen); + } + frame->ok = true; } /* txn_insert_jump_pad frees the entries jump read. */ @@ -3174,7 +3243,13 @@ txn_insert_jump_pad(void *frame_) /* * txn_insert_jump moves the counter from its load to a later tuple id by one * map-only prepare and a take of the counter block's entry, whose step says - * what the entry becomes, and answers what the take did. + * what the entry becomes, and answers what the take did. A replayed writer's + * take counts toward its transaction's free space map spill, as redo counts + * its rows, before the entry is read, so a spill it makes moves no version + * from under the entry the take falls back to; while the take answers that + * the entry or the base moved since the read, as a spill at its prepare's + * wait or a fold makes them, it reads the entry and prepares again (bugs row + * 342). */ static bool txn_insert_jump(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, TxnMemtableVersionWriter writer, @@ -3267,8 +3342,8 @@ txn_insert_Placed_read(HostIoIo *io, TreesStoreNodeStore store, const HostShared if (!txn_insert_seen(io, store, host_shared_Held_record(held), &result->base, &result->room, (uint32) (tid.f0 >> 16), &seen, error)) return false; - if (seen.has_current) - std_vec_free_Checked(&seen.current, 1, 1); + result->has_current = seen.has_current; + result->current = seen.current; result->has_below = seen.has_below; result->below = seen.below; return true; @@ -3282,49 +3357,243 @@ txn_insert_Placed_take(const TxnInsertPlaced *self, HostSharedRecord record) txn_free_space_most_len(self->room.most)); } -/* txn_insert_Placed_drop frees a place's base entry of its block. */ +/* txn_insert_Placed_drop frees a place's entries of its block. */ void txn_insert_Placed_drop(TxnInsertPlaced *self) { + if (self->has_current) + std_vec_free_Checked(&self->current, 1, 1); if (self->has_below) std_vec_free_Checked(&self->below, 1, 1); - self->has_below = false; + self->has_current = self->has_below = false; +} + +/* txn_insert_unfree returns the internal error of a carried version's place its storage's base doesn't hold free. */ +static HostError +txn_insert_unfree(uint64 tid) +{ + StdFmtPiece pieces[] = {STD_FMT_LIT("a carried version's place "), STD_FMT_ARG(0), + STD_FMT_LIT(" isn't free in its storage's base")}; + StdFmtArg args[] = {{&tid, std_u64_Display_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + StdString message; + + std_fmt_format_Checked(&message, &arguments); + return host_Error_fork((const uint8 *) "XX000", &message); } /* * txn_insert_place_taken takes a placed tuple id's place out of its block's * entry inside the window of the prepare whose take Placed_take readied it, * as the writer's, with redo's step, so redo of the row's record finds it - * taken; a place the entry doesn't hold free is an internal error. + * taken; a place the entry doesn't hold free is an internal error. It + * answers moved, taking nothing, when the block's entry or the base moved + * since Placed_read read them, so its caller reads the place again (bugs row + * 342). */ bool txn_insert_place_taken(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t take, - HostSharedRecord record, TxnMemtableVersionWriter writer, const TxnInsertPlaced *placed, - HostError *error) + HostSharedRecord record, TxnMemtableVersionWriter writer, bool has_roomed, + TxnStatusRoomed roomed, const TxnInsertPlaced *placed, bool *moved, HostError *error) { txn_insert_ReplayStep_t step = {.record = record, .base = &placed->base, .room = &placed->room, .counter = placed->counter, .tid = placed->tid}; + TxnMemtableMapRead read = {.has_entry = placed->has_current, .entry = placed->has_current ? placed->current.ptr : NULL, + .entry_len = placed->has_current ? placed->current.len : 0, .generation = placed->base.generation}; HostRow unclaimed = {0}; TxnMemtableMapTaken taken; - StdFmtPiece pieces[] = {STD_FMT_LIT("a carried version's place "), STD_FMT_ARG(0), - STD_FMT_LIT(" isn't free in its storage's base")}; - StdFmtArg args[] = {{&placed->tid, std_u64_Display_fmt}}; - StdFmtArguments arguments = {pieces, lengthof(pieces), args}; - StdString message; - if (!txn_memtable_map_took_Unit(io, prepared, take, writer, false, 0, false, unclaimed, 0, placed->has_below, - placed->below.ptr, placed->below.len, txn_insert_replayed, &step, NULL, + if (!txn_memtable_map_took_Unit(io, prepared, take, writer, has_roomed, roomed, false, 0, false, unclaimed, 0, + placed->has_below, placed->below.ptr, placed->below.len, &read, txn_insert_replayed, + &step, NULL, txn_insert_replayed_tuple, &step, NULL, txn_insert_nothing, NULL, NULL, &taken, error)) return false; - if (taken == TXN_MEMTABLE_MAP_TAKEN_TOOK) + *moved = taken == TXN_MEMTABLE_MAP_TAKEN_MOVED; + if (taken == TXN_MEMTABLE_MAP_TAKEN_TOOK || *moved) return true; - std_fmt_format_Checked(&message, &arguments); - *error = host_Error_fork((const uint8 *) "XX000", &message); + *error = txn_insert_unfree(placed->tid); return false; } -/* txn_insert_redo_spent_closure_0_captures_t is redo_spent's prepare's closure's captures. */ +/* txn_insert_carried_entry_t is carried_takes' block entry: the block and its entry after the takes so far, none for a removal. */ +typedef struct txn_insert_carried_entry_t +{ + uint32 block; + bool has_current; + StdVec current; +} txn_insert_carried_entry_t; + +/* txn_insert_carried_frame_t is carried_takes' state across its pad: the blocks' entries, a block's seen, and the takes, while live. */ +typedef struct txn_insert_carried_frame_t +{ + HostIoIo *io; + TreesStoreNodeStore store; + const HostSharedHeld *held; + const TxnInsertCarriedPut *puts; + size_t puts_len; + const TxnInsertPage *page; + StdVec *result; + HostError *error; + bool ok; + TxnInsertBase base; + StdVec entries; + bool entries_live; + TxnInsertSeen seen; + bool result_live; +} txn_insert_carried_frame_t; + +/* txn_insert_carried_pad frees what carried_takes holds live. */ +static void +txn_insert_carried_pad(void *frame_) +{ + txn_insert_carried_frame_t *frame = frame_; + + txn_insert_Seen_drop(&frame->seen); + if (frame->entries_live) + { + for (size_t k = 0; k < frame->entries.len; k++) + { + txn_insert_carried_entry_t *entry = &STD_VEC_AT(&frame->entries, txn_insert_carried_entry_t, k); + + if (entry->has_current) + std_vec_free_Checked(&entry->current, 1, 1); + } + std_vec_free_Checked(&frame->entries, sizeof(txn_insert_carried_entry_t), + _Alignof(txn_insert_carried_entry_t)); + } + if (frame->result_live) + txn_insert_carried_takes_drop(frame->result); + frame->entries_live = frame->result_live = false; +} + +/* txn_insert_carried_body takes each put's place after the take before it of its block, the first from the block's entry below the memtable. */ +static void +txn_insert_carried_body(void *frame_) +{ + txn_insert_carried_frame_t *frame = frame_; + HostSharedRecord record = host_shared_Held_record(frame->held); + uint64 counter = txn_state_counter(record); + + std_vec_new(frame->result, sizeof(TxnInsertCarriedTake), _Alignof(TxnInsertCarriedTake)); + frame->result_live = true; + std_vec_new(&frame->entries, sizeof(txn_insert_carried_entry_t), _Alignof(txn_insert_carried_entry_t)); + frame->entries_live = true; + if (!txn_insert_read_base(frame->io, frame->held, &frame->base, frame->error)) + return; + if (!std_vec_try_reserve_exact(frame->result, frame->puts_len, sizeof(TxnInsertCarriedTake), + _Alignof(TxnInsertCarriedTake))) + { + *frame->error = host_Error_out_of_memory(); + return; + } + for (size_t k = 0; k < frame->puts_len; k++) + { + const TxnInsertCarriedPut *put = &frame->puts[k]; + TxnInsertRoom room = {.bytes = put->heap_bytes, .page = frame->page->bytes, .most = frame->page->most}; + uint32 block = (uint32) (put->tid.f0 >> 16); + txn_insert_carried_entry_t *entry = NULL; + txn_insert_ReplayStep_t step = {.record = record, .base = &frame->base, .room = &room, .counter = counter, + .tid = put->tid.f0}; + TxnMemtableVersionShown command = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + TxnMemtableOp op = {0}; + TxnInsertCarriedTake take = {.block = block}; + StdVec value; + + for (size_t at = 0; at < frame->entries.len && entry == NULL; at++) + if (STD_VEC_AT(&frame->entries, txn_insert_carried_entry_t, at).block == block) + entry = &STD_VEC_AT(&frame->entries, txn_insert_carried_entry_t, at); + if (entry == NULL) + { + txn_insert_carried_entry_t seen = {.block = block}; + + if (!txn_insert_seen(frame->io, frame->store, record, &frame->base, &room, block, &frame->seen, + frame->error)) + return; + seen.has_current = frame->seen.has_current; + seen.current = frame->seen.current; + frame->seen.has_current = false; + txn_insert_Seen_drop(&frame->seen); + if (!std_vec_try_reserve(&frame->entries, 1, sizeof(seen), _Alignof(txn_insert_carried_entry_t))) + { + if (seen.has_current) + std_vec_free_Checked(&seen.current, 1, 1); + *frame->error = host_Error_out_of_memory(); + return; + } + std_vec_push_Checked(&frame->entries, &seen, sizeof(seen), _Alignof(txn_insert_carried_entry_t)); + entry = &STD_VEC_AT(&frame->entries, txn_insert_carried_entry_t, frame->entries.len - 1); + } + std_vec_new(&value, 1, 1); + switch (txn_insert_replayed(&step, entry->has_current, entry->current.ptr, entry->current.len, &value)) + { + case TXN_MEMTABLE_MAP_STEP_PUT: + op.tag = TXN_MEMTABLE_OP_PUT; + op.u.put.value = value; + std_vec_new(&op.u.put.addresses, sizeof(chunks_TupleU32Locator_t), _Alignof(chunks_TupleU32Locator_t)); + std_vec_new(&op.u.put.values, sizeof(uint32), _Alignof(uint32)); + op.u.put.tuple = put->tid; + break; + case TXN_MEMTABLE_MAP_STEP_REMOVE: + std_vec_free_Checked(&value, 1, 1); + op.tag = TXN_MEMTABLE_OP_EMPTIED; + op.u.emptied = put->tid; + break; + case TXN_MEMTABLE_MAP_STEP_RETRY: + case TXN_MEMTABLE_MAP_STEP_REFUSE: + std_vec_free_Checked(&value, 1, 1); + *frame->error = txn_insert_unfree(put->tid.f0); + return; + } + std_vec_new(&take.record, 1, 1); + txn_memtable_version_encode(&put->writer, &op, TXN_MEMTABLE_LOCK_MODE_UPDATE, command, false, 0, &take.record); + if (entry->has_current) + std_vec_free_Checked(&entry->current, 1, 1); + entry->has_current = op.tag == TXN_MEMTABLE_OP_PUT; + if (entry->has_current) + { + entry->current = op.u.put.value; + std_vec_free_Checked(&op.u.put.addresses, sizeof(chunks_TupleU32Locator_t), + _Alignof(chunks_TupleU32Locator_t)); + std_vec_free_Checked(&op.u.put.values, sizeof(uint32), _Alignof(uint32)); + } + std_vec_push_Checked(frame->result, &take, sizeof(take), _Alignof(TxnInsertCarriedTake)); + } + frame->result_live = false; + frame->ok = true; +} + +/* txn_insert_carried_takes_drop frees a carry's takes. */ +void +txn_insert_carried_takes_drop(StdVec *takes) +{ + for (size_t k = 0; k < takes->len; k++) + std_vec_free_Checked(&STD_VEC_AT(takes, TxnInsertCarriedTake, k).record, 1, 1); + std_vec_free_Checked(takes, sizeof(TxnInsertCarriedTake), _Alignof(TxnInsertCarriedTake)); +} + +/* + * txn_insert_carried_takes returns the free space map versions by which a + * carry's puts of other writers take their places in a new storage whose + * spill files they go to (P2), each put's block and its take's record, in + * the puts' order: each as place_taken takes it after the take before it of + * its block, the first from the block's entry below the memtable, so their + * chains read as the memtable's would have. + */ +bool +txn_insert_carried_takes(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, + const TxnInsertCarriedPut *puts, size_t puts_len, const TxnInsertPage *page, StdVec *result, + HostError *error) +{ + txn_insert_carried_frame_t frame = {.io = io, .store = store, .held = held, .puts = puts, .puts_len = puts_len, + .page = page, .result = result, .error = error}; + + std_landing(&frame, txn_insert_carried_body, txn_insert_carried_pad); + txn_insert_carried_pad(&frame); + return frame.ok; +} + +/* txn_insert_redo_spent_closure_0_captures_t is redo_spent's prepare's closure's captures, and what its take did. */ typedef struct txn_insert_redo_spent_closure_0_captures_t { TxnMemtableVersionWriter writer; @@ -3332,6 +3601,7 @@ typedef struct txn_insert_redo_spent_closure_0_captures_t txn_insert_ReplayStep_t *step; bool has_roomed; TxnStatusRoomed roomed; + TxnMemtableMapTaken taken; } txn_insert_redo_spent_closure_0_captures_t; /* txn_insert_redo_spent_moved is redo_spent's take's then: the counter moved past a tuple id at or past it. */ @@ -3344,23 +3614,20 @@ txn_insert_redo_spent_moved(void *captures) txn_state_moved(step->record, step->counter, STD_ADD(u64, step->tid, 1, 1479, 55)); } -/* txn_insert_redo_spent_closure_0 is redo_spent's prepare's closure: the place's take, counted when it took. */ +/* txn_insert_redo_spent_closure_0 is redo_spent's prepare's closure: the place's take. */ static bool txn_insert_redo_spent_closure_0(void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, HostError *error) { txn_insert_redo_spent_closure_0_captures_t *inside = captures; HostRow unclaimed = {0}; - TxnMemtableMapTaken taken; + TxnMemtableMapRead read = txn_insert_Seen_read(inside->seen, inside->step->base); - if (!txn_memtable_map_took_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, inside->writer, false, 0, false, - unclaimed, 0, inside->seen->has_below, inside->seen->below.ptr, - inside->seen->below.len, txn_insert_replayed, inside->step, NULL, - txn_insert_replayed_tuple, inside->step, NULL, txn_insert_redo_spent_moved, - inside->step, NULL, &taken, error)) - return false; - return taken != TXN_MEMTABLE_MAP_TAKEN_TOOK || - txn_memtable_table_counted(HOST_IO_IO_TAKES_SHORT(io), prepared, inside->has_roomed, inside->roomed, error); + return txn_memtable_map_took_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, inside->writer, inside->has_roomed, + inside->roomed, false, 0, false, unclaimed, 0, inside->seen->has_below, + inside->seen->below.ptr, inside->seen->below.len, &read, txn_insert_replayed, + inside->step, NULL, txn_insert_replayed_tuple, inside->step, NULL, + txn_insert_redo_spent_moved, inside->step, NULL, &inside->taken, error); } /* txn_insert_redo_spent_frame_t is redo_spent's state across its pad: the block's entries it read. */ @@ -3391,7 +3658,7 @@ txn_insert_redo_spent_body(void *frame_) uint32 block; txn_insert_ReplayStep_t step; txn_insert_redo_spent_closure_0_captures_t inside = {.writer = frame->spent->writer, .seen = &frame->seen, - .step = &step}; + .step = &step, .taken = TXN_MEMTABLE_MAP_TAKEN_TOOK}; TxnMemtableMapTake take; if (!txn_insert_read_base(frame->io, frame->held, &frame->base, frame->error)) @@ -3418,9 +3685,15 @@ txn_insert_redo_spent_body(void *frame_) step.room = &frame->room; step.counter = counter; step.tid = tid; - frame->ok = txn_memtable_map_prepare_Unit(frame->io, &take, 1, txn_insert_redo_spent_closure_0, &inside, - NULL, frame->error); - return; + if (!txn_memtable_map_prepare_Unit(frame->io, &take, 1, txn_insert_redo_spent_closure_0, &inside, NULL, + frame->error)) + return; + if (inside.taken != TXN_MEMTABLE_MAP_TAKEN_MOVED) + { + frame->ok = true; + return; + } + txn_insert_Seen_drop(&frame->seen); } } @@ -3436,7 +3709,9 @@ txn_insert_redo_spent_pad(void *frame_) /* * txn_insert_redo_spent applies a replayed spent place, with no catalog * access: it takes the place out of the free space map as the primary's - * take did, as redo takes a row's, and pushes no version. + * take did, as redo takes a row's, and pushes no version, reading the + * block's entry and preparing again while its take answers that the entry or + * the base moved since the read (bugs row 342). */ bool txn_insert_redo_spent(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, @@ -3580,6 +3855,13 @@ typedef struct txn_insert_redo_frame_t uint64 tid; size_t need; bool pushed; + + /* + * mark is the state's mark the entries were read at, and done whether the + * window found it unmoved. + */ + std_TupleU64U64_t mark; + bool done; StdVec stored; bool stored_live; StdVec kept; @@ -3599,7 +3881,11 @@ typedef struct txn_insert_redo_frame_t bool ok; } txn_insert_redo_frame_t; -/* txn_insert_redo_refuse is a replayed stored value's step: its entry where the map holds none, else nothing. */ +/* + * txn_insert_redo_refuse is a replayed stored value's step: its entry where + * the map holds none, else nothing; as txn_insert_stored's, its take has no + * read to compare (bugs row 342). + */ static TxnMemtableMapStep txn_insert_redo_refuse(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out) { @@ -3623,23 +3909,29 @@ txn_insert_redo_closure_9(void *captures, HostIoIo *io_, TxnMemtableTablePrepare TxnMemtableMapTaken taken; TxnMemtableVersionHandle handle; StdString message; + std_TupleU64U64_t mark = txn_values_mark(step.record); + TxnMemtableMapRead read = txn_insert_Seen_read(&frame->seen, &frame->base); - if (!txn_memtable_map_took_Unit(io, prepared, 0, change->writer, false, 0, false, unclaimed, 0, - frame->seen.has_below, frame->seen.below.ptr, frame->seen.below.len, - txn_insert_replayed, &step, NULL, txn_insert_replayed_tuple, &step, NULL, - txn_insert_replayed_moved, &step, NULL, &taken, error)) - return false; - if (taken == TXN_MEMTABLE_MAP_TAKEN_TOOK && - !txn_memtable_table_counted(io, prepared, frame->has_roomed, frame->roomed, error)) + frame->done = mark.f0 == frame->mark.f0 && mark.f1 == frame->mark.f1; + if (!frame->done) + return true; + if (!txn_memtable_map_took_Unit(io, prepared, 0, change->writer, frame->has_roomed, frame->roomed, false, 0, false, + unclaimed, 0, frame->seen.has_below, frame->seen.below.ptr, + frame->seen.below.len, &read, txn_insert_replayed, &step, NULL, + txn_insert_replayed_tuple, &step, NULL, txn_insert_replayed_moved, &step, NULL, + &taken, error)) return false; + if (taken == TXN_MEMTABLE_MAP_TAKEN_MOVED) + { + frame->done = false; + return true; + } for (size_t at = 0; frame->toast != NULL && at < frame->entries_len; at++) { - if (!txn_memtable_map_finish_Unit(io, prepared, at + 1, change->writer, false, 0, false, NULL, 0, - txn_insert_redo_refuse, (void *) &frame->entries[at], NULL, - txn_insert_nothing, NULL, NULL, &taken, error)) - return false; - if (taken == TXN_MEMTABLE_MAP_TAKEN_TOOK && - !txn_memtable_table_counted(io, prepared, frame->has_roomed, frame->roomed, error)) + if (!txn_memtable_map_finish_Unit(io, prepared, at + 1, change->writer, frame->has_roomed, frame->roomed, + false, 0, false, NULL, 0, NULL, txn_insert_redo_refuse, + (void *) &frame->entries[at], NULL, txn_insert_nothing, NULL, NULL, &taken, + error)) return false; } for (size_t at = 0; at < frame->kept.len; at++) @@ -3647,8 +3939,9 @@ txn_insert_redo_closure_9(void *captures, HostIoIo *io_, TxnMemtableTablePrepare const txn_insert_TupleU32OptionVecU8_t *kept = &STD_VEC_AT(&frame->kept, txn_insert_TupleU32OptionVecU8_t, at); - if (!txn_memtable_map_finish_Unit(io, prepared, 1 + frame->stored.len + at, change->writer, true, frame->lsn, - kept->has_f1, kept->f1.ptr, kept->f1.len, txn_insert_added, NULL, NULL, + if (!txn_memtable_map_finish_Unit(io, prepared, 1 + frame->stored.len + at, change->writer, + frame->has_roomed, frame->roomed, true, frame->lsn, kept->has_f1, + kept->f1.ptr, kept->f1.len, NULL, txn_insert_added, NULL, NULL, txn_insert_nothing, NULL, NULL, &taken, error)) return false; if (taken != TXN_MEMTABLE_MAP_TAKEN_TOOK) @@ -3657,17 +3950,14 @@ txn_insert_redo_closure_9(void *captures, HostIoIo *io_, TxnMemtableTablePrepare *error = host_Error_fork((const uint8 *) "XX001", &message); return false; } - if (!txn_memtable_table_counted(io, prepared, frame->has_roomed, frame->roomed, error)) - return false; } if (frame->marks.len > 0) { StdVec marks = frame->marks; std_vec_new(&frame->marks, TXN_INSERT_ID); - if (!txn_memtable_map_named(io, prepared, frame->marking, change->writer, true, frame->lsn, row, &marks, - error) || - !txn_memtable_table_counted(io, prepared, frame->has_roomed, frame->roomed, error)) + if (!txn_memtable_map_named(io, prepared, frame->marking, change->writer, frame->has_roomed, frame->roomed, + true, frame->lsn, row, &marks, error)) return false; } if (!frame->pushed) @@ -3683,14 +3973,28 @@ txn_insert_redo_closure_9(void *captures, HostIoIo *io_, TxnMemtableTablePrepare return false; if (frame->has_flagged) { - txn_insert_TakeStep_t filled = {.record = step.record, .base = &frame->base, .room = &frame->room, + TxnInsertBase flagged_base = frame->base; + txn_insert_TakeStep_t filled = {.record = step.record, .base = &flagged_base, .room = &frame->room, .block = frame->flagged}; - if (!txn_memtable_map_finish_Unit(io, prepared, frame->takes.len - 1, change->writer, false, 0, - frame->flagged_seen.has_below, frame->flagged_seen.below.ptr, - frame->flagged_seen.below.len, txn_insert_filled, &filled, NULL, - txn_insert_nothing, NULL, NULL, &taken, error)) - return false; + for (;;) + { + TxnMemtableMapRead flagged = txn_insert_Seen_read(&frame->flagged_seen, &flagged_base); + + if (!txn_memtable_map_finish_Unit(io, prepared, frame->takes.len - 1, change->writer, frame->has_roomed, + frame->roomed, false, 0, frame->flagged_seen.has_below, + frame->flagged_seen.below.ptr, frame->flagged_seen.below.len, &flagged, + txn_insert_filled, &filled, NULL, txn_insert_nothing, NULL, NULL, &taken, + error)) + return false; + if (taken != TXN_MEMTABLE_MAP_TAKEN_MOVED) + break; + txn_insert_Seen_drop(&frame->flagged_seen); + if (!txn_insert_read_base(frame->io, frame->held, &flagged_base, error) || + !txn_insert_seen(frame->io, frame->store, step.record, &flagged_base, &frame->room, frame->flagged, + &frame->flagged_seen, error)) + return false; + } if (taken == TXN_MEMTABLE_MAP_TAKEN_RETRY) { std_string_from_Checked(&message, "a free space map's entry is corrupt", 35); @@ -3771,6 +4075,20 @@ txn_insert_redo_body(void *frame_) !txn_spill_spilled(frame->io, &change->storage, TXN_WRITTEN_OF_ROWS, top, &spilled, frame->error)) return; } + if (!frame->pushed) + { + size_t frees = 1 + (change->has_replaced && change->replaced.f0 >> 16 != frame->tid >> 16); + size_t freeing = frees * (txn_free_space_most_len(frame->room.most) + 4); + uint64 bytes; + bool spilled; + + if (!txn_spill_replayed(frame->io, &change->storage, TXN_WRITTEN_OF_FREE, top, freeing, &bytes, + frame->error)) + return; + if (bytes >= txn_spill_threshold() && + !txn_spill_spilled(frame->io, &change->storage, TXN_WRITTEN_OF_FREE, top, &spilled, frame->error)) + return; + } for (;;) { uint32 block; @@ -3787,8 +4105,10 @@ txn_insert_redo_body(void *frame_) continue; } block = (uint32) (frame->tid >> 16); + frame->mark = txn_values_mark(record); if (!txn_insert_seen(frame->io, frame->store, record, &frame->base, &frame->room, block, &frame->seen, - frame->error)) + frame->error) || + !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-redo-take-seen", frame->error)) return; std_vec_new(&frame->takes, TXN_INSERT_TAKE); frame->takes_live = true; @@ -3836,9 +4156,22 @@ txn_insert_redo_body(void *frame_) row.f0.ptr = change->key.ptr; row.f0.len = change->key.len; row.f1 = frame->need; - frame->ok = txn_memtable_table_prepare_Unit(frame->io, record, &row, 1, frame->takes.ptr, frame->takes.len, - true, txn_insert_redo_closure_9, frame, NULL, frame->error); - return; + if (!txn_memtable_table_prepare_Unit(frame->io, record, &row, 1, frame->takes.ptr, frame->takes.len, true, + txn_insert_redo_closure_9, frame, NULL, frame->error)) + return; + if (frame->done) + { + frame->ok = true; + return; + } + if (frame->has_flagged) + txn_insert_Seen_drop(&frame->flagged_seen); + frame->has_flagged = false; + std_vec_free_Checked(&frame->marks, TXN_INSERT_ID); + frame->marks_live = false; + std_vec_free_Checked(&frame->takes, TXN_INSERT_TAKE); + frame->takes_live = false; + txn_insert_Seen_drop(&frame->seen); } } @@ -3884,7 +4217,10 @@ txn_insert_redo_pad(void *frame_) * values has each value's map entry written unless the map holds the id * already. A row version the checkpoint's runs brought back isn't pushed * again, while the record's map changes and its tuple id's slot apply either - * way. + * way. The entries a take falls back to are read before its prepare, which + * may spill at the delta's budget or meet a fold's install; a window that + * finds the state's mark moved since writes nothing and reads them again, so + * no take compares against an entry a spill moved out of the memtable. */ bool txn_insert_redo(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, const TxnRecordRowChange *change, @@ -3899,3 +4235,4167 @@ txn_insert_redo(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *h txn_insert_redo_pad(&frame); return frame.ok; } + +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/chunks/trailer.h" +#include "doltgres/repo/lib.h" +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" +#include "doltgres/txn/blobs.h" +#include "doltgres/txn/change.h" +#include "doltgres/txn/memtable/branch.h" +#include "doltgres/txn/record.h" +#include "doltgres/txn/runs.h" +#include "doltgres/txn/values.h" +#include "doltgres/txn/visible.h" + +/* + * txn_insert_tests_taken inserts a row as a test does, taking its key's + * stripe with a test's Waits and no resolve, and returns its tuple id. + */ +bool +txn_insert_tests_taken(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, TxnInsertInsert *row, + const TxnInsertRoom *room, const HostSharedHeld *toast, const TxnValuesEntry *toast_entries, + size_t toast_entries_len, uint64 *result, HostError *error) +{ + static const StdFmtPiece pieces[] = {STD_FMT_LIT("a test's insert gave "), STD_FMT_ARG(0)}; + HostIoWaits waits = host_io_Waits_testing(); + TxnInsertKeyed keyed = {.stripe = txn_change_stripe(&row->storage, row->key.ptr, row->key.len, true)}; + TxnInsertInserted inserted; + TxnInsertInsertedRef other = {&inserted, {NULL, std_unit_Debug_fmt}}; + StdFmtArg args[] = {{&other, txn_insert_Inserted_Debug_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + StdString message; + + if (!txn_insert_insert(io, store, held, row, room, NULL, toast, toast_entries, toast_entries_len, keyed, &waits, + &inserted, error)) + return false; + if (inserted.tag == TXN_INSERT_INSERTED_TOOK) + { + *result = inserted.u.took.tuple; + return true; + } + std_fmt_format_Checked(&message, &arguments); + *error = host_Error_fork((const uint8 *) "XX000", &message); + return false; +} + +/* txn_insert_tests_stored returns a test's storage in pg_default, by its database. */ +HostStorage +txn_insert_tests_stored(uint32 database) +{ + return (HostStorage) + { + .tablespace = HOST_PG_DEFAULT, .database = database, .relation = 16384 + }; +} + +/* txn_insert_tests_row returns a keyed row's insert by its key's number, written by a transaction's first command. */ +TxnInsertInsert +txn_insert_tests_row(HostStorage storage, uint64 n) +{ + TxnInsertInsert row = {.storage = storage, .table = 16385, .put = {.heap_bytes = 32}, .writer = {.xid = 41}, + .logged = true}; + uint8 key[8]; + + std_u64_to_be_bytes(n, key); + std_slice_to_vec_Checked(&row.key, key, sizeof(key), 1, 1); + std_slice_to_vec_Checked(&row.put.value, (const uint8 *) "row", 3, 1, 1); + std_vec_new(&row.put.addresses, sizeof(chunks_TupleU32Locator_t), _Alignof(chunks_TupleU32Locator_t)); + std_vec_new(&row.put.values, sizeof(uint32), _Alignof(uint32)); + std_vec_new(&row.kept, sizeof(size_t), _Alignof(size_t)); + return row; +} + +/* TXN_INSERT_TESTS_UNWRAP_FAILED is a Result's unwrap's panic on a HostError at a line and column. */ +#define TXN_INSERT_TESTS_UNWRAP_FAILED(line, col, error) \ + STD_RESULT_EXPECT_FAILED((line), (col), "called `Result::unwrap()` on an `Err` value", (error), \ + host_Error_Debug_fmt) + +/* TXN_INSERT_TESTS_PAGE is an 8 kB heap page as redo knows it: its bytes past its header and its most tuples. */ +static const TxnInsertPage TXN_INSERT_TESTS_PAGE = {.bytes = 8168, .most = 291}; + +/* txn_insert_tests_room returns what a row of some heap bytes needs of an 8 kB heap page, with no fill factor's reserve. */ +static TxnInsertRoom +txn_insert_tests_room(uint64 bytes) +{ + TxnInsertRoom room = {.bytes = bytes, .page = 8168, .most = 291}; + + return room; +} + +/* txn_insert_tests_repository makes a test's repository of a database and a reader of the shared state, unwrapped at the lines a case gives. */ +static void +txn_insert_tests_repository(HostIoIo *io, uint32 database, uint32 create_line, uint32 create_col, uint32 read_line, + uint32 read_col, RepoRepo *repo, HostSharedReader *reading) +{ + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_create(io, database, HOST_PG_DEFAULT, repo, &failed)) + STD_RESULT_EXPECT_FAILED(create_line, create_col, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!host_shared_read(io, reading, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(read_line, read_col, &error); +} + +/* txn_insert_tests_stated returns the state of a storage, unwrapped at a line and column. */ +static HostSharedHeld +txn_insert_tests_stated(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, HostStorage storage, + uint32 line, uint32 col) +{ + HostSharedHeld held; + HostError error; + + if (!txn_state_state(io, repo, reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &held, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(line, col, &error); + return held; +} + +/* txn_insert_tests_took inserts a row as taken does, unwrapped at a line and column, and returns its tuple id. */ +static uint64 +txn_insert_tests_took(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, TxnInsertInsert row, + uint64 bytes, const HostSharedHeld *toast, const TxnValuesEntry *entries, size_t entries_len, + uint32 line, uint32 col) +{ + TxnInsertRoom room = txn_insert_tests_room(bytes); + HostError error; + uint64 tid; + + if (!txn_insert_tests_taken(io, store, held, &row, &room, toast, entries, entries_len, &tid, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(line, col, &error); + return tid; +} + +/* txn_insert_tests_ended ends the test's transaction as its commit's callbacks do, unwrapped at a line and column. */ +static void +txn_insert_tests_ended(HostIoIo *io, uint32 line, uint32 col) +{ + HostError error; + + if (!txn_written_ended(io, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(line, col, &error); +} + +/* txn_insert_tests_inserted returns the tuple ids a new storage's inserts of a row's heap bytes take, one row a key. */ +static StdVec +txn_insert_tests_inserted(HostIoIo *io, uint32 database, uint64 bytes, uint64 rows) +{ + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + StdVec tids; + + txn_insert_tests_repository(io, database, 1740, 91, 1741, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(database), 1742, 110); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + std_vec_with_capacity_Checked(&tids, rows, sizeof(uint64), _Alignof(uint64)); + for (uint64 n = 0; n < rows; n++) + { + uint64 tid = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(database), n), bytes, + NULL, NULL, 0, 1745, 91); + + std_vec_push_Checked(&tids, &tid, sizeof(tid), _Alignof(uint64)); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return tids; +} + +/* txn_insert_tests_u64s_is asserts at a line and column that some u64s are those wanted, as slices compare. */ +static void +txn_insert_tests_u64s_is(uint32 line, uint32 col, const uint64 *got, size_t got_len, const uint64 *want, + size_t want_len) +{ + StdVec left = {(void *) got, got_len, got_len}; + StdVec right = {(void *) want, want_len, want_len}; + + STD_ASSERT_EQ(line, col, got_len == want_len && memcmp(got, want, got_len * sizeof(uint64)) == 0, &left, + chunks_trailer_VecU64_Debug_fmt, &right, chunks_trailer_VecU64_Debug_fmt); +} + +/* + * PLAN item 2's allocation on a new table, heap's packing: rows of 28 heap + * bytes take (0,1) up to (0,291), a page's most tuples, then (1,1); rows of + * 240 heap bytes fill a page by its bytes at its 34th row, and the 35th goes + * to (1,1). + */ +void +test_txn_insert_tests_inserts_take_heaps_tuple_ids_by_offsets_and_by_bytes(void) +{ + static const uint64 first[] = {1, 2}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + StdVec narrow = txn_insert_tests_inserted(io, 9301, 28, 292); + StdVec wide; + + txn_insert_tests_u64s_is(1756, 9, narrow.ptr, 2, first, lengthof(first)); + STD_ASSERT_EQ_U64(1757, 9, STD_VEC_AT(&narrow, uint64, 290), 291); + STD_ASSERT_EQ_U64(1758, 9, STD_VEC_AT(&narrow, uint64, 291), 1 << 16 | 1); + wide = txn_insert_tests_inserted(io, 9302, 240, 35); + STD_ASSERT_EQ_U64(1760, 9, STD_VEC_AT(&wide, uint64, 33), 34); + STD_ASSERT_EQ_U64(1761, 9, STD_VEC_AT(&wide, uint64, 34), 1 << 16 | 1); + txn_insert_tests_ended(io, 1762, 35); + std_vec_free_Checked(&wide, sizeof(uint64), _Alignof(uint64)); + std_vec_free_Checked(&narrow, sizeof(uint64), _Alignof(uint64)); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_last_page_t is the last page case's captures: its storage, its state and store, the rows put, and the block the last took. */ +typedef struct txn_insert_tests_last_page_t +{ + HostStorage storage; + TreesStoreNodeStore store; + const HostSharedHeld *held; + uint64 n; + bool has_target; + uint32 target; +} txn_insert_tests_last_page_t; + +/* txn_insert_tests_last_page_put is the last page case's put: the next key's row of some heap bytes, at the block the last row took. */ +static uint64 +txn_insert_tests_last_page_put(txn_insert_tests_last_page_t *c, HostIoIo *io, uint64 bytes) +{ + TxnInsertInsert row; + uint64 tid; + + c->n++; + row = txn_insert_tests_row(c->storage, c->n); + row.has_target = c->has_target; + row.target = c->target; + tid = txn_insert_tests_took(io, c->store, c->held, row, bytes, NULL, NULL, 0, 1781, 93); + c->has_target = true; + c->target = (uint32) (tid >> 16); + return tid; +} + +/* + * PLAN item 2's allocation in RelationGetBufferForTuple's order, with a fresh + * page left by its bytes handing its counter offsets to the map: rows of 3000 + * heap bytes take (0,1), (0,2), then (1,1), (1,2), (2,1); rows that fit the + * last page go there, (2,2) and (2,3); and a row the last page can't hold + * goes to the lowest page whose room the map shows, the page left first, at + * its first counter offset, (0,3). + */ +void +test_txn_insert_tests_a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room(void) +{ + static const uint64 sizes[] = {3000, 3000, 3000, 3000, 3000, 2000, 3000, 2000}; + static const uint64 want[] = {1, 2, 1 << 16 | 1, 1 << 16 | 2, 2 << 16 | 1, 2 << 16 | 2, 2 << 16 | 3, 3}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + txn_insert_tests_last_page_t c = {.storage = txn_insert_tests_stored(9303)}; + uint64 taken[lengthof(sizes)]; + + txn_insert_tests_repository(io, 9303, 1773, 87, 1774, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, c.storage, 1775, 95); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + c.store = TREES_STORE_READER_NODE_STORE(&store); + c.held = &held; + for (size_t k = 0; k < lengthof(sizes); k++) + taken[k] = txn_insert_tests_last_page_put(&c, io, sizes[k]); + txn_insert_tests_u64s_is(1787, 9, taken, lengthof(taken), want, lengthof(want)); + txn_insert_tests_ended(io, 1788, 35); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_logged_tag_t is Logged's variants. */ +typedef enum txn_insert_tests_logged_tag_t +{ + /* A row change with its record's end. */ + TXN_INSERT_TESTS_LOGGED_ROW, + /* A fresh page's leave. */ + TXN_INSERT_TESTS_LOGGED_LEFT, + /* A chunk with its record's end. */ + TXN_INSERT_TESTS_LOGGED_CHUNK, +} txn_insert_tests_logged_tag_t; + +/* + * txn_insert_tests_logged_t is Logged: one of a database's logged writes, as + * redo reads it back, a row change with its record's end, a fresh page's + * leave, or a chunk with its record's end. + */ +typedef struct txn_insert_tests_logged_t +{ + txn_insert_tests_logged_tag_t tag; + union + { + TxnRecordRowChange row; + TxnRecordPageLeft left; + TxnRecordChunkWrite chunk; + } u; + HostLsn end; +} txn_insert_tests_logged_t; + +/* txn_insert_tests_logged_drop drops a logged write. */ +static void +txn_insert_tests_logged_drop(txn_insert_tests_logged_t *write) +{ + if (write->tag == TXN_INSERT_TESTS_LOGGED_ROW) + txn_record_RowChange_drop(&write->u.row); + else if (write->tag == TXN_INSERT_TESTS_LOGGED_CHUNK) + txn_record_ChunkWrite_drop(&write->u.chunk); +} + +/* txn_insert_tests_logged_free frees a Vec and its writes. */ +static void +txn_insert_tests_logged_free(StdVec *writes) +{ + for (size_t k = 0; k < writes->len; k++) + txn_insert_tests_logged_drop(&STD_VEC_AT(writes, txn_insert_tests_logged_t, k)); + std_vec_free_Checked(writes, sizeof(txn_insert_tests_logged_t), _Alignof(txn_insert_tests_logged_t)); +} + +/* txn_insert_tests_logged returns the writes the simulation logged for a database's storages and its repository, in their order. */ +static StdVec +txn_insert_tests_logged(uint32 database) +{ + StdVec logged = simc_logged(); + StdVec writes; + + std_vec_new(&writes, sizeof(txn_insert_tests_logged_t), _Alignof(txn_insert_tests_logged_t)); + for (size_t k = 0; k < logged.len; k++) + { + simc_TupleU8VecU8Lsn_t *one = &STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, k); + txn_insert_tests_logged_t write = {.end = one->f2}; + HostWalKind kind; + HostError error; + bool kept = false; + + if (!host_WalKind_of(one->f0, &kind)) + kind = HOST_WAL_KIND_REPOSITORY_CREATED; + if (kind == HOST_WAL_KIND_ROW_CHANGE) + { + write.tag = TXN_INSERT_TESTS_LOGGED_ROW; + if (!txn_record_RowChange_decode(one->f1.ptr, one->f1.len, &write.u.row, &error)) + host_Error_drop(&error); + else if (!(kept = write.u.row.storage.database == database)) + txn_record_RowChange_drop(&write.u.row); + } + else if (kind == HOST_WAL_KIND_PAGE_LEFT) + { + write.tag = TXN_INSERT_TESTS_LOGGED_LEFT; + if (!txn_record_PageLeft_decode(one->f1.ptr, one->f1.len, &write.u.left, &error)) + host_Error_drop(&error); + else + kept = write.u.left.storage.database == database; + } + else if (kind == HOST_WAL_KIND_CHUNK) + { + write.tag = TXN_INSERT_TESTS_LOGGED_CHUNK; + if (!txn_record_ChunkWrite_decode(one->f1.ptr, one->f1.len, &write.u.chunk, &error)) + host_Error_drop(&error); + else if (!(kept = write.u.chunk.repository == database)) + txn_record_ChunkWrite_drop(&write.u.chunk); + } + if (kept) + std_vec_push_Checked(&writes, &write, sizeof(write), _Alignof(txn_insert_tests_logged_t)); + std_vec_free_Checked(&one->f1, 1, 1); + } + std_vec_free_Checked(&logged, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + return writes; +} + +/* txn_insert_tests_count returns how many of some logged writes are of a variant. */ +static size_t +txn_insert_tests_count(const StdVec *writes, txn_insert_tests_logged_tag_t tag) +{ + size_t count = 0; + + for (size_t k = 0; k < writes->len; k++) + count += STD_VEC_AT(writes, txn_insert_tests_logged_t, k).tag == tag; + return count; +} + +/* txn_insert_tests_mapped_t is an Option>>: a key's newest map entry, a removal's none, or none. */ +typedef struct txn_insert_tests_mapped_t +{ + bool some; + TxnMemtableMapEntry entry; +} txn_insert_tests_mapped_t; + +/* txn_insert_tests_mapped_Debug_fmt is an Option>>'s Debug. */ +static bool +txn_insert_tests_mapped_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_mapped_t *mapped = value; + StdBytes bytes = {mapped->entry.value.ptr, mapped->entry.value.len}; + StdOptionRef inner = {mapped->some && mapped->entry.has_value, {&bytes, std_bytes_Debug_fmt}}; + StdOptionRef outer = {mapped->some, {&inner, std_option_Debug_fmt}}; + + return std_option_Debug_fmt(&outer, f); +} + +/* txn_insert_tests_mapped_eq reports whether two Option>>s are equal. */ +static bool +txn_insert_tests_mapped_eq(const txn_insert_tests_mapped_t *a, const txn_insert_tests_mapped_t *b) +{ + return a->some == b->some && + (!a->some || (a->entry.has_value == b->entry.has_value && + (!a->entry.has_value || + trees_store_VecU8_PartialEq_eq(&a->entry.value, &b->entry.value)))); +} + +/* txn_insert_tests_mapped_drop drops a map entry's value. */ +static void +txn_insert_tests_mapped_drop(txn_insert_tests_mapped_t *mapped) +{ + if (mapped->some && mapped->entry.has_value) + std_vec_free_Checked(&mapped->entry.value, 1, 1); +} + +/* txn_insert_tests_newest returns a map's newest entry of a key, unwrapped at a line and column. */ +static txn_insert_tests_mapped_t +txn_insert_tests_newest(HostIoIo *io, HostSharedRecord record, TxnMemtableMapMap map, uint32 key, uint32 line, + uint32 col) +{ + txn_insert_tests_mapped_t mapped = {0}; + HostError error; + + if (!txn_memtable_map_newest(io, record, map, key, &mapped.some, &mapped.entry, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(line, col, &error); + return mapped; +} + +/* txn_insert_tests_free_t is a state's free space map entries of its first four blocks, and its counter. */ +typedef struct txn_insert_tests_free_t +{ + txn_insert_tests_mapped_t entries[4]; + uint64 counter; +} txn_insert_tests_free_t; + +/* txn_insert_tests_free_entries returns a state's free space map entries of its first four blocks, with its counter. */ +static txn_insert_tests_free_t +txn_insert_tests_free_entries(HostIoIo *io, HostSharedRecord record) +{ + txn_insert_tests_free_t free; + + for (uint32 block = 0; block < 4; block++) + free.entries[block] = txn_insert_tests_newest(io, record, TXN_MEMTABLE_MAP_MAP_FREE, block, 1821, 70); + free.counter = txn_state_counter(record); + return free; +} + +/* txn_insert_tests_mappeds_Debug_fmt is a Vec>>>'s Debug. */ +static bool +txn_insert_tests_mappeds_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *mappeds = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < mappeds->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(mappeds, txn_insert_tests_mapped_t, k), + txn_insert_tests_mapped_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_insert_tests_free_Debug_fmt is the Debug of a state's first four blocks' entries with its counter, the tuple (entries, counter). */ +static bool +txn_insert_tests_free_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_free_t *free = value; + StdVec entries = {(void *) free->entries, lengthof(free->entries), lengthof(free->entries)}; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&entries, txn_insert_tests_mappeds_Debug_fmt), + STD_ITEM(&free->counter, std_u64_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_insert_tests_free_drop drops a state's first four blocks' entries. */ +static void +txn_insert_tests_free_drop(txn_insert_tests_free_t *free) +{ + for (size_t k = 0; k < lengthof(free->entries); k++) + txn_insert_tests_mapped_drop(&free->entries[k]); +} + +/* + * txn_insert_tests_free_is asserts at a line and column, with a message or + * none, that a replica's entries and counter are the primary's. + */ +static void +txn_insert_tests_free_is(uint32 line, uint32 col, const txn_insert_tests_free_t *got, + const txn_insert_tests_free_t *want, const StdFmtArguments *message) +{ + bool eq = got->counter == want->counter; + + for (size_t k = 0; eq && k < lengthof(got->entries); k++) + eq = txn_insert_tests_mapped_eq(&got->entries[k], &want->entries[k]); + if (!eq) + std_assert_failed(RUST_FILE, line, col, "==", got, txn_insert_tests_free_Debug_fmt, want, + txn_insert_tests_free_Debug_fmt, message); +} + +/* + * txn_insert_tests_replica_t is Replica: a new database's storage a + * primary's logged writes were redone on, its state, its TOAST relation's + * state when a row change named one, and its repository's working state's + * delta of chunks. + */ +typedef struct txn_insert_tests_replica_t +{ + HostSharedHeld held; + bool has_toast; + HostSharedHeld toast; + HostSharedHeld deltas; +} txn_insert_tests_replica_t; + +/* txn_insert_tests_replica_drop drops a replica's states and its delta of chunks. */ +static void +txn_insert_tests_replica_drop(txn_insert_tests_replica_t *replica) +{ + host_shared_Held_drop(&replica->held); + if (replica->has_toast) + host_shared_Held_drop(&replica->toast); + host_shared_Held_drop(&replica->deltas); +} + +/* + * txn_insert_tests_redone redoes a primary's logged writes, in their given + * order, on a state as the fork's resource manager does: each row change with + * its TOAST relation's state when it named one, each leave, and each chunk + * into deltas. It takes the writes. + */ +static void +txn_insert_tests_redone(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, + const HostSharedHeld *toast, const HostSharedHeld *deltas, StdVec *writes) +{ + for (size_t k = 0; k < writes->len; k++) + { + txn_insert_tests_logged_t *write = &STD_VEC_AT(writes, txn_insert_tests_logged_t, k); + HostError error; + + if (write->tag == TXN_INSERT_TESTS_LOGGED_ROW) + { + const TxnRecordRowChange *change = &write->u.row; + const HostSharedHeld *named = change->has_toast ? toast : NULL; + StdVec entries; + bool ok; + + std_vec_with_capacity_Checked(&entries, change->blobs.len, sizeof(TxnValuesEntry), + _Alignof(TxnValuesEntry)); + for (size_t b = 0; b < change->blobs.len; b++) + { + const chunks_TupleAddressU64_t *blob = &STD_VEC_AT(&change->blobs, chunks_TupleAddressU64_t, b); + TxnValuesEntry entry = txn_values_Entry_new(blob->f0, blob->f1); + + std_vec_push_Checked(&entries, &entry, sizeof(entry), _Alignof(TxnValuesEntry)); + } + ok = txn_insert_redo(io, store, held, change, write->end, &TXN_INSERT_TESTS_PAGE, named, + named == NULL ? NULL : entries.ptr, named == NULL ? 0 : entries.len, &error); + std_vec_free_Checked(&entries, sizeof(TxnValuesEntry), _Alignof(TxnValuesEntry)); + if (!ok) + TXN_INSERT_TESTS_UNWRAP_FAILED(1870, 71, &error); + txn_record_RowChange_drop(&write->u.row); + } + else if (write->tag == TXN_INSERT_TESTS_LOGGED_LEFT) + { + if (!txn_insert_redo_left(io, store, held, &write->u.left, &TXN_INSERT_TESTS_PAGE, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(1872, 80, &error); + } + else if (!txn_blobs_redo(io, deltas, write->u.chunk, write->end, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(1873, 82, &error); + } + std_vec_free_Checked(writes, sizeof(txn_insert_tests_logged_t), _Alignof(txn_insert_tests_logged_t)); +} + +/* + * txn_insert_tests_replica redoes a primary's logged writes, in their given + * order, on a new database's storage with no catalog, as the fork's resource + * manager does: each row change with its TOAST relation's state and the + * value map entries its field 24 gives, each leave, and each chunk into the + * new repository's working state. It takes the writes. + */ +static txn_insert_tests_replica_t +txn_insert_tests_replica(HostIoIo *io, uint32 database, StdVec *writes) +{ + txn_insert_tests_replica_t replica = {0}; + RepoRepo repo; + HostSharedReader reading; + TreesStoreReader store; + HostError error; + + txn_insert_tests_repository(io, database, 1836, 91, 1837, 46, &repo, &reading); + replica.held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(database), 1838, 110); + for (size_t k = 0; !replica.has_toast && k < writes->len; k++) + { + const txn_insert_tests_logged_t *write = &STD_VEC_AT(writes, txn_insert_tests_logged_t, k); + + if (write->tag == TXN_INSERT_TESTS_LOGGED_ROW && write->u.row.has_toast) + { + HostStorage storage = {.tablespace = write->u.row.toast.tablespace, .database = database, + .relation = write->u.row.toast.relation}; + + replica.toast = txn_insert_tests_stated(io, &repo, &reading, storage, 1845, 94); + replica.has_toast = true; + } + } + if (!txn_memtable_branch_deltas(io, database, HOST_PG_DEFAULT, &replica.deltas, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(1847, 97, &error); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + txn_insert_tests_redone(io, TREES_STORE_READER_NODE_STORE(&store), &replica.held, + replica.has_toast ? &replica.toast : NULL, &replica.deltas, writes); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return replica; +} + +/* + * txn_insert_tests_replayed redoes a primary's writes on a new database's + * replica and compares its entries and counter with the primary's, with a + * message naming the replica when named. + */ +static void +txn_insert_tests_replayed(HostIoIo *io, uint32 database, StdVec *writes, const txn_insert_tests_free_t *expected, + bool named, uint32 line, uint32 col) +{ + static const StdFmtPiece pieces[] = {STD_FMT_LIT("replica "), STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&database, std_u32_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + txn_insert_tests_replica_t replica = txn_insert_tests_replica(io, database, writes); + txn_insert_tests_free_t got; + + if (replica.has_toast) + { + host_shared_Held_drop(&replica.toast); + replica.has_toast = false; + } + host_shared_Held_drop(&replica.deltas); + got = txn_insert_tests_free_entries(io, host_shared_Held_record(&replica.held)); + txn_insert_tests_free_is(line, col, &got, expected, named ? &message : NULL); + txn_insert_tests_free_drop(&got); + host_shared_Held_drop(&replica.held); +} + +/* + * CommPF 5's redo cases: the primary's inserts, replayed on another storage + * from their logged records with no catalog, leave every block's free space + * entry and the counter as the primary's, the reference placement's case + * among them, whose last row goes to (0,3) after page 2 is left; in the + * records' order, and with two row changes logged out of their tuple ids' + * order, as two inserts' windows allow. + */ +void +test_txn_insert_tests_redo_leaves_the_free_space_map_and_the_counter_as_the_primarys(void) +{ + static const uint64 sizes[] = {3000, 3000, 3000, 3000, 3000, 2000, 3000, 2000}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo primary; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + bool has_target = false; + uint32 target = 0; + txn_insert_tests_free_t expected; + StdVec writes; + StdVec swapped; + + txn_insert_tests_repository(io, 9304, 1888, 95, 1889, 50, &primary, &reading); + held = txn_insert_tests_stated(io, &primary, &reading, txn_insert_tests_stored(9304), 1893, 109); + store = repo_Repo_reader(&primary, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (uint64 n = 0; n < lengthof(sizes); n++) + { + TxnInsertInsert row = txn_insert_tests_row(txn_insert_tests_stored(9304), n); + uint64 tid; + + row.has_target = has_target; + row.target = target; + tid = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, sizes[n], NULL, NULL, 0, + 1898, 104); + has_target = true; + target = (uint32) (tid >> 16); + } + expected = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + writes = txn_insert_tests_logged(9304); + STD_ASSERT_EQ_U64(1902, 9, txn_insert_tests_count(&writes, TXN_INSERT_TESTS_LOGGED_LEFT), 3); + txn_insert_tests_logged_free(&writes); + swapped = txn_insert_tests_logged(9304); + { + txn_insert_tests_logged_t first = STD_VEC_AT(&swapped, txn_insert_tests_logged_t, 0); + + STD_VEC_AT(&swapped, txn_insert_tests_logged_t, 0) = STD_VEC_AT(&swapped, txn_insert_tests_logged_t, 1); + STD_VEC_AT(&swapped, txn_insert_tests_logged_t, 1) = first; + } + writes = txn_insert_tests_logged(9304); + txn_insert_tests_replayed(io, 9305, &writes, &expected, true, 1908, 13); + txn_insert_tests_replayed(io, 9306, &swapped, &expected, true, 1908, 13); + txn_insert_tests_ended(io, 1910, 35); + txn_insert_tests_free_drop(&expected); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&primary); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_appended moves the writes of one vector onto the end of another. */ +static void +txn_insert_tests_appended(StdVec *to, StdVec *from) +{ + std_vec_extend_from_slice_Checked(to, from->ptr, from->len, sizeof(txn_insert_tests_logged_t), + _Alignof(txn_insert_tests_logged_t)); + std_vec_free_Checked(from, sizeof(txn_insert_tests_logged_t), _Alignof(txn_insert_tests_logged_t)); +} + +/* + * PLAN's I12 for a free space take: the reference placement's inserts redone + * with each record twice in a row, and redone whole and then whole again, as + * a checkpoint's runs that hold a window's take can be followed by its + * record, leave every block's free space entry and the counter as the + * primary's, since a take whose offset is no longer free and not past the + * counter changes nothing. + */ +void +test_txn_insert_tests_a_take_redone_again_changes_nothing(void) +{ + static const uint64 sizes[] = {3000, 3000, 3000, 3000, 3000, 2000, 3000, 2000}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + txn_insert_tests_free_t expected; + StdVec a; + StdVec b; + StdVec twice; + StdVec again; + + txn_insert_tests_repository(io, 9307, 1920, 87, 1921, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9307), 1922, 106); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (uint64 n = 0; n < lengthof(sizes); n++) + txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9307), n), sizes[n], NULL, NULL, 0, + 1926, 73); + expected = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + a = txn_insert_tests_logged(9307); + b = txn_insert_tests_logged(9307); + std_vec_with_capacity_Checked(&twice, a.len + b.len, sizeof(txn_insert_tests_logged_t), + _Alignof(txn_insert_tests_logged_t)); + for (size_t k = 0; k < a.len && k < b.len; k++) + { + std_vec_push_Checked(&twice, &STD_VEC_AT(&a, txn_insert_tests_logged_t, k), sizeof(txn_insert_tests_logged_t), + _Alignof(txn_insert_tests_logged_t)); + std_vec_push_Checked(&twice, &STD_VEC_AT(&b, txn_insert_tests_logged_t, k), sizeof(txn_insert_tests_logged_t), + _Alignof(txn_insert_tests_logged_t)); + } + std_vec_free_Checked(&a, sizeof(txn_insert_tests_logged_t), _Alignof(txn_insert_tests_logged_t)); + std_vec_free_Checked(&b, sizeof(txn_insert_tests_logged_t), _Alignof(txn_insert_tests_logged_t)); + again = txn_insert_tests_logged(9307); + b = txn_insert_tests_logged(9307); + txn_insert_tests_appended(&again, &b); + txn_insert_tests_replayed(io, 9308, &twice, &expected, true, 1934, 13); + txn_insert_tests_replayed(io, 9309, &again, &expected, true, 1934, 13); + txn_insert_tests_ended(io, 1936, 35); + txn_insert_tests_free_drop(&expected); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_toasted returns the state of a test's TOAST relation 16390 in a database, unwrapped at a line and column. */ +static HostSharedHeld +txn_insert_tests_toasted(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, uint32 database, + uint32 line, uint32 col) +{ + HostStorage storage = txn_insert_tests_stored(database); + + storage.relation = 16390; + return txn_insert_tests_stated(io, repo, reading, storage, line, col); +} + +/* TXN_INSERT_TESTS_TOAST is the TOAST relation a test's valued rows name: pg_default's 16390, of table 16391. */ +static const TxnRecordToast TXN_INSERT_TESTS_TOAST = {.tablespace = HOST_PG_DEFAULT, .relation = 16390, .table = 16391}; + +/* TXN_INSERT_TESTS_IDS is the value ids a test's valued rows name. */ +static const uint32 TXN_INSERT_TESTS_IDS[] = {16400, 16401}; + +/* txn_insert_tests_values sets an insert's value ids to the test's two, naming its TOAST relation. */ +static void +txn_insert_tests_values(TxnInsertInsert *row) +{ + std_vec_free_Checked(&row->put.values, sizeof(uint32), _Alignof(uint32)); + std_slice_to_vec_Checked(&row->put.values, TXN_INSERT_TESTS_IDS, lengthof(TXN_INSERT_TESTS_IDS), sizeof(uint32), + _Alignof(uint32)); + row->has_toast = true; + row->toast = TXN_INSERT_TESTS_TOAST; +} + +/* txn_insert_tests_row_changes returns pointers to the row changes among some logged writes, in their order. */ +static StdVec +txn_insert_tests_row_changes(const StdVec *writes) +{ + StdVec rows; + + std_vec_new(&rows, sizeof(const TxnRecordRowChange *), _Alignof(const TxnRecordRowChange *)); + for (size_t k = 0; k < writes->len; k++) + { + const txn_insert_tests_logged_t *write = &STD_VEC_AT(writes, txn_insert_tests_logged_t, k); + const TxnRecordRowChange *row = &write->u.row; + + if (write->tag == TXN_INSERT_TESTS_LOGGED_ROW) + std_vec_push_Checked(&rows, &row, sizeof(row), _Alignof(const TxnRecordRowChange *)); + } + return rows; +} + +/* + * Rule (C) of the out-of-line values text: an insert of a row with two + * out-of-line values writes each value's map entry, with a count of one, + * into its TOAST relation's value map in the row's window, and logs the + * value ids with their TOAST relation in the row's record (fields 24 to 27). + */ +void +test_txn_insert_tests_an_insert_maps_its_out_of_line_values_in_its_window(void) +{ + static const char *const bytes[] = {"one", "two"}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld toast; + TreesStoreReader store; + TxnValuesEntry entries[2]; + TxnInsertInsert valued; + StdVec writes; + StdVec logged; + StdVec ids = {(void *) TXN_INSERT_TESTS_IDS, 2, 2}; + + txn_insert_tests_repository(io, 9311, 1946, 87, 1947, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9311), 1948, 106); + toast = txn_insert_tests_toasted(io, &repo, &reading, 9311, 1950, 102); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (size_t k = 0; k < lengthof(entries); k++) + { + entries[k].address = chunks_address_Address_of((const uint8 *) bytes[k], strlen(bytes[k])); + entries[k].locator = chunks_trailer_Locator_Default_default(); + entries[k].raw = 4000; + entries[k].stored = 4000; + entries[k].count = 1; + } + valued = txn_insert_tests_row(txn_insert_tests_stored(9311), 1); + txn_insert_tests_values(&valued); + txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, valued, 60, &toast, entries, + lengthof(entries), 1963, 75); + for (size_t k = 0; k < lengthof(entries); k++) + { + txn_insert_tests_mapped_t mapped = txn_insert_tests_newest(io, host_shared_Held_record(&toast), + TXN_MEMTABLE_MAP_MAP_VALUES, + TXN_INSERT_TESTS_IDS[k], 1965, 75); + TxnValuesEntry entry; + HostError error; + + if (!mapped.some) + STD_OPTION_UNWRAP_FAILED(1965, 84); + if (!mapped.entry.has_value) + STD_OPTION_UNWRAP_FAILED(1965, 93); + if (!txn_values_Entry_decode(mapped.entry.value.ptr, mapped.entry.value.len, &entry, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(1966, 48, &error); + STD_ASSERT_EQ(1966, 13, txn_values_Entry_PartialEq_eq(&entry, &entries[k]), &entry, + txn_values_Entry_Debug_fmt, &entries[k], txn_values_Entry_Debug_fmt); + txn_insert_tests_mapped_drop(&mapped); + } + writes = txn_insert_tests_logged(9311); + logged = txn_insert_tests_row_changes(&writes); + STD_ASSERT_EQ_U64(1974, 9, logged.len, 1); + { + const TxnRecordRowChange *change = STD_VEC_AT(&logged, const TxnRecordRowChange *, 0); + const TxnMemtablePut *put; + StdOptionRef got = {change->has_toast, {&change->toast, txn_record_Toast_Debug_fmt}}; + StdOptionRef want = {true, {&TXN_INSERT_TESTS_TOAST, txn_record_Toast_Debug_fmt}}; + + if (!txn_record_RowChange_put(change, &put)) + STD_OPTION_UNWRAP_FAILED(1975, 36); + STD_ASSERT_EQ(1975, 9, put->values.len == 2 && + memcmp(put->values.ptr, TXN_INSERT_TESTS_IDS, sizeof(TXN_INSERT_TESTS_IDS)) == 0, + &put->values, chunks_header_VecU32_Debug_fmt, &ids, chunks_header_VecU32_Debug_fmt); + STD_ASSERT_EQ(1976, 9, change->has_toast && + txn_record_Toast_PartialEq_eq(&change->toast, &TXN_INSERT_TESTS_TOAST), &got, + std_option_Debug_fmt, &want, std_option_Debug_fmt); + } + std_vec_free_Checked(&logged, sizeof(const TxnRecordRowChange *), _Alignof(const TxnRecordRowChange *)); + txn_insert_tests_logged_free(&writes); + txn_insert_tests_ended(io, 1977, 35); + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_linked reports whether a state's rows' memtable, free space map, and tuple id table are each linked. */ +static void +txn_insert_tests_linked(HostSharedRecord record, bool linked[3]) +{ + static const HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + + linked[0] = host_shared_Record_atomic(record, TXN_STATE_MEMTABLE_WORD_INDEX, load) != 0; + linked[1] = host_shared_Record_atomic(record, TXN_STATE_FREE_WORD_INDEX, load) != 0; + linked[2] = host_shared_Record_atomic(record, TXN_STATE_TUPLES_WORD_INDEX, load) != 0; +} + +/* txn_insert_tests_gathering_fork returns a gathering case's actor's error, the fork's internal one, of a formatted message. */ +static HostError +txn_insert_tests_gathering_fork(const StdFmtArguments *arguments) +{ + StdString message; + + std_fmt_format_Checked(&message, arguments); + return host_Error_fork((const uint8 *) "XX000", &message); +} + +/* txn_insert_tests_opened opens a storage's database's repository, a reader, and the storage's state, dropping what it opened on a failure. */ +static bool +txn_insert_tests_opened(HostIoIo *io, HostStorage storage, RepoRepo *repo, HostSharedReader *reading, HostSharedHeld *held, + HostError *error) +{ + if (!txn_state_tests_reopened(io, storage.database, repo, error)) + return false; + if (!host_shared_read(io, reading, error)) + { + repo_Repo_drop(repo); + return false; + } + if (!txn_state_state(io, repo, reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, held, error)) + { + host_shared_Reader_drop(reading); + repo_Repo_drop(repo); + return false; + } + return true; +} + +/* txn_insert_tests_gathering_inserter is the gathering case's inserter: its row takes (0,1). */ +static bool +txn_insert_tests_gathering_inserter(void *captures, HostIoIo *io, HostError *error) +{ + static const StdFmtPiece pieces[] = {STD_FMT_LIT("the insert took "), STD_FMT_ARG(0)}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TxnInsertRoom room = txn_insert_tests_room(28); + uint64 tid; + bool ok; + + if (!txn_insert_tests_opened(io, txn_insert_tests_stored(9312), &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + TxnInsertInsert row = txn_insert_tests_row(txn_insert_tests_stored(9312), 1); + + ok = txn_insert_tests_taken(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &room, NULL, NULL, 0, + &tid, error); + } + if (ok && tid != 1) + { + StdFmtArg args[] = {{&tid, std_u64_Display_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + *error = txn_insert_tests_gathering_fork(&arguments); + ok = false; + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* txn_insert_tests_gathering_between is the gathering case's step between a save's manifest write and its saved, which does nothing. */ +static void +txn_insert_tests_gathering_between(void *captures, HostIoIo *io) +{ +} + +/* txn_insert_tests_found_t is what a gathering found: whether the state's three words are linked, and its counter. */ +typedef struct txn_insert_tests_found_t +{ + bool linked[3]; + uint64 counter; +} txn_insert_tests_found_t; + +/* txn_insert_tests_linked_Debug_fmt is a [bool; 3]'s Debug. */ +static bool +txn_insert_tests_linked_Debug_fmt(const void *value, StdFormatter *f) +{ + const bool *linked = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < 3; k++) + std_DebugList_entry(&list, &linked[k], std_bool_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_insert_tests_found_Debug_fmt is what a gathering found's Debug, the tuple ([bool; 3], u64). */ +static bool +txn_insert_tests_found_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_found_t *found = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(found->linked, txn_insert_tests_linked_Debug_fmt), + STD_ITEM(&found->counter, std_u64_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_insert_tests_gathering_gatherer is one of the gathering case's gatherers: a save's gathering finds the state's words linked and its counter where the insert put it. */ +static bool +txn_insert_tests_gathering_gatherer(void *captures, HostIoIo *io, HostError *error) +{ + static const StdFmtPiece pieces[] = {STD_FMT_LIT("a gathering found "), STD_FMT_ARG(0)}; + uint64 counter = *(const uint64 *) captures; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + txn_insert_tests_found_t found; + bool ok = true; + + if (!txn_insert_tests_opened(io, txn_insert_tests_stored(9312), &repo, &reading, &held, error)) + return false; + txn_state_tests_saved(io, &repo, 9312, txn_insert_tests_gathering_between, NULL); + txn_insert_tests_linked(host_shared_Held_record(&held), found.linked); + found.counter = txn_state_counter(host_shared_Held_record(&held)); + if (!(found.linked[0] && found.linked[1] && found.linked[2] && found.counter == counter)) + { + StdFmtArg args[] = {{&found, txn_insert_tests_found_Debug_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + *error = txn_insert_tests_gathering_fork(&arguments); + ok = false; + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * PLAN item 7's tuple id counter: a save's gathering held at each point from + * an insert's prepare to its push, at the prepare's pause before the take, at + * the window's pause between the row's record and its push, and at the + * settle's pause after the push, finds the storage's rows' memtable, free + * space map, and tuple id table linked, and keeps the counter where the + * insert put it, so the row's tuple id is never handed out again. + */ +void +test_txn_insert_tests_a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter(void) +{ + static const uint64 counters[] = {1, 2, 2}; + static const char *const names[] = {"before", "logged", "pushed"}; + static const SimcStep before[] = {{"before", 6, UINT64_MAX}}; + static const SimcStep logged[] = {{"logged", 6, UINT64_MAX}}; + static const SimcStep pushed[] = {{"pushed", 6, UINT64_MAX}, {"inserter", 8, UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + SimcActor inserter = {txn_insert_tests_gathering_inserter}; + RepoRepo repo; + ChunksErrorError failed; + SimcRun run; + SimcReport report; + + if (!repo_Repo_create(io, 9312, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(1999, 81, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + repo_Repo_drop(&repo); + run = simc_Run_new(163); + simc_Run_actor(&run, "inserter", 8, inserter); + for (size_t k = 0; k < lengthof(counters); k++) + { + SimcActor gatherer = {txn_insert_tests_gathering_gatherer, (void *) &counters[k]}; + + simc_Run_actor(&run, names[k], strlen(names[k]), gatherer); + } + simc_Run_until(&run, "inserter", 8, "doltgres-memtable-prepared"); + simc_Run_script(&run, before, lengthof(before)); + simc_Run_until(&run, "inserter", 8, "doltgres-insert-logged"); + simc_Run_script(&run, logged, lengthof(logged)); + simc_Run_until(&run, "inserter", 8, "doltgres-memtable-unpinned"); + simc_Run_script(&run, pushed, lengthof(pushed)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, RUST_FILE, 2029, 67, 2030, 9); + simc_Report_drop(&report); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_failed_is asserts at a line and column that an insert failed with the fork's internal error, and drops the error. */ +static void +txn_insert_tests_failed_is(uint32 line, uint32 col, bool failed, HostError *error) +{ + bool internal = failed && memcmp(host_Error_sqlstate(error), "XX000", 5) == 0; + + if (failed) + host_Error_drop(error); + STD_ASSERT(line, col, internal, "failed.is_err_and(|error| error.sqlstate() == b\"XX000\")"); +} + +/* + * The leave case: an insert that leaves the fresh page and then fails before + * its row's record, a crash, and redo of the logged records on another + * storage leave every block's free space entry and the counter as the + * primary's, since the leave's record hands the page's counter offsets to the + * map and the failed turn logged nothing. + */ +void +test_txn_insert_tests_an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + TxnInsertRoom room = txn_insert_tests_room(3000); + TxnInsertInsert row; + HostError error; + uint64 tid; + bool failed; + txn_insert_tests_free_t expected; + StdVec writes; + + txn_insert_tests_repository(io, 9313, 2041, 87, 2042, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9313), 2043, 106); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (uint64 n = 0; n < 2; n++) + txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9313), n), 3000, NULL, NULL, 0, + 2047, 73); + simc_refuse_any_take(1); + row = txn_insert_tests_row(txn_insert_tests_stored(9313), 2); + failed = !txn_insert_tests_taken(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &room, NULL, NULL, 0, + &tid, &error); + simc_refuse_nothing(); + txn_insert_tests_failed_is(2052, 9, failed, &error); + writes = txn_insert_tests_logged(9313); + STD_ASSERT_EQ_U64(2054, 9, txn_insert_tests_count(&writes, TXN_INSERT_TESTS_LOGGED_ROW), 2); + STD_ASSERT_EQ_U64(2055, 9, txn_insert_tests_count(&writes, TXN_INSERT_TESTS_LOGGED_LEFT), 1); + expected = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + STD_ASSERT_EQ_U64(2057, 9, expected.counter, 1 << 16 | 1); + txn_insert_tests_replayed(io, 9314, &writes, &expected, false, 2059, 9); + txn_insert_tests_ended(io, 2060, 35); + txn_insert_tests_free_drop(&expected); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_mapped returns a TOAST relation's value map entries for the test's two value ids. */ +static void +txn_insert_tests_mapped(HostIoIo *io, const HostSharedHeld *toast, txn_insert_tests_mapped_t mapped[2]) +{ + for (size_t k = 0; k < lengthof(TXN_INSERT_TESTS_IDS); k++) + mapped[k] = txn_insert_tests_newest(io, host_shared_Held_record(toast), TXN_MEMTABLE_MAP_MAP_VALUES, + TXN_INSERT_TESTS_IDS[k], 2065, 79); +} + +/* + * txn_insert_tests_valued returns an insert of a row with two out-of-line + * values whose references sit in an overflow chunk, so the row's own bytes + * hold only that chunk's address, with each value's map entry. + */ +static TxnInsertInsert +txn_insert_tests_valued(uint32 database, TxnValuesEntry entries[2]) +{ + static const char *const bytes[] = {"one", "two"}; + static const uint64 lengths[] = {4000, 300000}; + TxnInsertInsert valued = txn_insert_tests_row(txn_insert_tests_stored(database), 1); + ChunksAddressAddress overflow = chunks_address_Address_of((const uint8 *) "overflow", 8); + chunks_TupleU32Locator_t address = {.f0 = 0, .f1 = chunks_trailer_Locator_Default_default()}; + + std_vec_free_Checked(&valued.put.value, 1, 1); + std_slice_to_vec_Checked(&valued.put.value, overflow.f0, sizeof(overflow.f0), 1, 1); + std_vec_free_Checked(&valued.put.addresses, sizeof(chunks_TupleU32Locator_t), _Alignof(chunks_TupleU32Locator_t)); + std_slice_to_vec_Checked(&valued.put.addresses, &address, 1, sizeof(chunks_TupleU32Locator_t), + _Alignof(chunks_TupleU32Locator_t)); + txn_insert_tests_values(&valued); + for (size_t k = 0; k < 2; k++) + entries[k] = txn_values_Entry_new(chunks_address_Address_of((const uint8 *) bytes[k], 3), lengths[k]); + return valued; +} + +/* txn_insert_tests_valued_t is a storage's first four blocks' free space entries with its TOAST relation's entries of the test's two value ids. */ +typedef struct txn_insert_tests_valued_t +{ + txn_insert_tests_mapped_t entries[4]; + txn_insert_tests_mapped_t values[2]; +} txn_insert_tests_valued_t; + +/* txn_insert_tests_valued_Debug_fmt is the Debug of a storage's entries with its value map's, the tuple of their two lists. */ +static bool +txn_insert_tests_valued_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_valued_t *valued = value; + StdVec entries = {(void *) valued->entries, 4, 4}; + StdVec values = {(void *) valued->values, 2, 2}; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&entries, txn_insert_tests_mappeds_Debug_fmt), + STD_ITEM(&values, txn_insert_tests_mappeds_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_insert_tests_valued_of returns a storage's first four blocks' entries with its TOAST relation's two value entries. */ +static txn_insert_tests_valued_t +txn_insert_tests_valued_of(HostIoIo *io, const HostSharedHeld *held, const HostSharedHeld *toast) +{ + txn_insert_tests_free_t free = txn_insert_tests_free_entries(io, host_shared_Held_record(held)); + txn_insert_tests_valued_t valued; + + memcpy(valued.entries, free.entries, sizeof(valued.entries)); + txn_insert_tests_mapped(io, toast, valued.values); + return valued; +} + +/* txn_insert_tests_valued_drop drops a storage's entries and its value map's. */ +static void +txn_insert_tests_valued_drop(txn_insert_tests_valued_t *valued) +{ + for (size_t k = 0; k < 4; k++) + txn_insert_tests_mapped_drop(&valued->entries[k]); + for (size_t k = 0; k < 2; k++) + txn_insert_tests_mapped_drop(&valued->values[k]); +} + +/* txn_insert_tests_row_is asserts at a line and column that a tuple id table's row is key 1's. */ +static void +txn_insert_tests_row_is(uint32 line, uint32 col, bool some, const StdVec *row) +{ + static const uint8 one[8] = {0, 0, 0, 0, 0, 0, 0, 1}; + StdBytes got = {some ? row->ptr : NULL, some ? row->len : 0}; + StdBytes want = {one, sizeof(one)}; + StdOptionRef left = {some, {&got, std_bytes_Debug_fmt}}; + StdOptionRef right = {true, {&want, std_bytes_Debug_fmt}}; + + STD_ASSERT_EQ(line, col, some && row->len == sizeof(one) && memcmp(row->ptr, one, sizeof(one)) == 0, &left, + std_option_Debug_fmt, &right, std_option_Debug_fmt); +} + +/* + * The overflow chunk case: a row whose out-of-line values' references sit in + * an overflow chunk, so its own bytes name none, redone after a crash from + * its record's field 24 alone, leaves its tuple id, its free space entries, + * and its TOAST relation's value map entries as the primary's. + */ +void +test_txn_insert_tests_a_row_whose_value_references_sit_in_an_overflow_chunk_redoes_its_map_entries(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld toast; + TreesStoreReader store; + TxnValuesEntry entries[2]; + TxnInsertInsert row; + uint64 tid; + HostRow tuple; + txn_insert_tests_valued_t expected; + txn_insert_tests_valued_t got; + txn_insert_tests_replica_t replica; + StdVec writes; + StdVec tupled; + HostError error; + bool some; + bool eq = true; + + txn_insert_tests_repository(io, 9315, 2088, 87, 2089, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9315), 2090, 106); + toast = txn_insert_tests_toasted(io, &repo, &reading, 9315, 2099, 10); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + row = txn_insert_tests_valued(9315, entries); + tid = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, 60, &toast, entries, 2, + 2103, 82); + expected = txn_insert_tests_valued_of(io, &held, &toast); + STD_ASSERT(2105, 9, expected.values[0].some && expected.values[0].entry.has_value && + expected.values[1].some && expected.values[1].entry.has_value, + "expected.1.iter().all(|entry| matches!(entry, Some(Some(_))))"); + writes = txn_insert_tests_logged(9315); + replica = txn_insert_tests_replica(io, 9316, &writes); + if (!replica.has_toast) + STD_OPTION_UNWRAP_FAILED(2107, 47); + got = txn_insert_tests_valued_of(io, &replica.held, &replica.toast); + for (size_t k = 0; eq && k < 4; k++) + eq = txn_insert_tests_mapped_eq(&got.entries[k], &expected.entries[k]); + for (size_t k = 0; eq && k < 2; k++) + eq = txn_insert_tests_mapped_eq(&got.values[k], &expected.values[k]); + STD_ASSERT_EQ(2108, 9, eq, &got, txn_insert_tests_valued_Debug_fmt, &expected, + txn_insert_tests_valued_Debug_fmt); + tuple.f0 = tid; + if (!txn_memtable_tuples_get(io, host_shared_Held_record(&replica.held), tuple, &some, &tupled, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2109, 70, &error); + txn_insert_tests_row_is(2109, 9, some, &tupled); + if (some) + std_vec_free_Checked(&tupled, 1, 1); + txn_insert_tests_ended(io, 2110, 35); + txn_insert_tests_valued_drop(&got); + txn_insert_tests_replica_drop(&replica); + txn_insert_tests_valued_drop(&expected); + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_unpacked is written's answer for a chunk's packing: no chunk is in a pack yet. */ +static bool +txn_insert_tests_unpacked(void *captures, HostIoIo *io, const ChunksAddressAddress *address, bool *result, + HostError *error) +{ + *result = false; + return true; +} + +/* + * txn_insert_tests_written writes a row's two out-of-line values' chunks + * into its repository's working state, logged, under the row's writer, as the + * fork's insert entry does before the row's window, and returns their working + * state. + */ +static HostSharedHeld +txn_insert_tests_written(HostIoIo *io, uint32 database, TxnMemtableVersionWriter writer, + const TxnValuesEntry entries[2]) +{ + static const char *const bytes[] = {"one", "two"}; + HostSharedHeld deltas; + HostError error; + StdVec chunks; + bool named = true; + + if (!txn_memtable_branch_deltas(io, database, HOST_PG_DEFAULT, &deltas, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2117, 73, &error); + std_vec_with_capacity_Checked(&chunks, 2, sizeof(txn_TupleAddressVecU8_t), _Alignof(txn_TupleAddressVecU8_t)); + for (size_t k = 0; k < 2; k++) + { + txn_TupleAddressVecU8_t chunk = {.f0 = chunks_address_Address_of((const uint8 *) bytes[k], 3)}; + + std_slice_to_vec_Checked(&chunk.f1, (const uint8 *) bytes[k], 3, 1, 1); + std_vec_push_Checked(&chunks, &chunk, sizeof(chunk), _Alignof(txn_TupleAddressVecU8_t)); + } + for (size_t k = 0; named && k < 2; k++) + named = memcmp(STD_VEC_AT(&chunks, txn_TupleAddressVecU8_t, k).f0.f0, entries[k].address.f0, + sizeof(entries[k].address.f0)) == 0; + STD_ASSERT(2119, 9, named, "chunks.iter().zip(entries).all(|((address, _), entry)| *address == entry.address)"); + if (!txn_blobs_written(io, &deltas, database, writer, &chunks, true, txn_insert_tests_unpacked, NULL, NULL, + &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2120, 91, &error); + return deltas; +} + +/* txn_insert_tests_mappeds_is asserts at a line and column, with a message or none, that two value map entries are those wanted. */ +static void +txn_insert_tests_mappeds_is(uint32 line, uint32 col, const txn_insert_tests_mapped_t got[2], + const txn_insert_tests_mapped_t want[2], const StdFmtArguments *message) +{ + StdVec left = {(void *) got, 2, 2}; + StdVec right = {(void *) want, 2, 2}; + + if (!txn_insert_tests_mapped_eq(&got[0], &want[0]) || !txn_insert_tests_mapped_eq(&got[1], &want[1])) + std_assert_failed(RUST_FILE, line, col, "==", &left, txn_insert_tests_mappeds_Debug_fmt, &right, + txn_insert_tests_mappeds_Debug_fmt, message); +} + +/* txn_insert_tests_durable reports whether a chunk of a repository's working state is durable, unwrapped at a line and column. */ +static bool +txn_insert_tests_durable(HostIoIo *io, const HostSharedHeld *deltas, const TxnValuesEntry *entry, uint32 line, + uint32 col) +{ + HostError error; + bool durable; + + if (!txn_memtable_branch_durable(io, host_shared_Held_record(deltas), entry->address.f0, + sizeof(entry->address.f0), &durable, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(line, col, &error); + return durable; +} + +/* + * Rule (C)'s first case: an insert with two out-of-line values that fails + * after its chunks' records and before its window leaves no value map entry + * and logs no row change, and its chunks stay in the working state + * unreferenced. + */ +void +test_txn_insert_tests_an_insert_that_fails_after_its_chunks_maps_no_value(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld toast; + HostSharedHeld deltas; + TreesStoreReader store; + TxnValuesEntry entries[2]; + TxnInsertInsert row; + TxnInsertRoom room = txn_insert_tests_room(60); + txn_insert_tests_mapped_t mapped[2]; + txn_insert_tests_mapped_t none[2] = {0}; + HostError error; + uint64 tid; + bool failed; + StdVec writes; + + txn_insert_tests_repository(io, 9317, 2130, 87, 2131, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9317), 2132, 106); + toast = txn_insert_tests_toasted(io, &repo, &reading, 9317, 2141, 10); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + row = txn_insert_tests_valued(9317, entries); + deltas = txn_insert_tests_written(io, 9317, row.writer, entries); + simc_refuse_any_take(0); + failed = !txn_insert_tests_taken(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &room, &toast, entries, + 2, &tid, &error); + simc_refuse_nothing(); + txn_insert_tests_failed_is(2149, 9, failed, &error); + txn_insert_tests_mapped(io, &toast, mapped); + txn_insert_tests_mappeds_is(2150, 9, mapped, none, NULL); + writes = txn_insert_tests_logged(9317); + STD_ASSERT_EQ_U64(2152, 9, txn_insert_tests_count(&writes, TXN_INSERT_TESTS_LOGGED_CHUNK), 2); + STD_ASSERT(2153, 9, txn_insert_tests_count(&writes, TXN_INSERT_TESTS_LOGGED_ROW) == 0, + "!writes.iter().any(|write| matches!(write, Logged::Row(..)))"); + for (size_t k = 0; k < 2; k++) + STD_ASSERT(2155, 13, txn_insert_tests_durable(io, &deltas, &entries[k], 2155, 76), + "branch::durable(io, deltas.record(), &entry.address.0).unwrap()"); + txn_insert_tests_ended(io, 2157, 35); + txn_insert_tests_logged_free(&writes); + for (size_t k = 0; k < 2; k++) + txn_insert_tests_mapped_drop(&mapped[k]); + host_shared_Held_drop(&deltas); + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_from_t is what a replica redone from a point holds: its row by the tuple id, and its two value map entries. */ +typedef struct txn_insert_tests_from_t +{ + bool some; + StdVec row; + txn_insert_tests_mapped_t values[2]; +} txn_insert_tests_from_t; + +/* txn_insert_tests_from_Debug_fmt is the Debug of what a replica holds, the tuple (row, values). */ +static bool +txn_insert_tests_from_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_from_t *from = value; + StdBytes row = {from->row.ptr, from->row.len}; + StdOptionRef some = {from->some, {&row, std_bytes_Debug_fmt}}; + StdVec values = {(void *) from->values, 2, 2}; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&some, std_option_Debug_fmt), + STD_ITEM(&values, txn_insert_tests_mappeds_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_insert_tests_from_eq reports whether what two replicas hold is equal. */ +static bool +txn_insert_tests_from_eq(const txn_insert_tests_from_t *a, const txn_insert_tests_from_t *b) +{ + return a->some == b->some && (!a->some || trees_store_VecU8_PartialEq_eq(&a->row, &b->row)) && + txn_insert_tests_mapped_eq(&a->values[0], &b->values[0]) && + txn_insert_tests_mapped_eq(&a->values[1], &b->values[1]); +} + +/* + * Rule (C)'s second case, as far as the logged records show it: redo started + * at each R across an insert with two out-of-line values, before its chunks' + * records, between them and its row change, and after the row change, gives + * the row with every value map entry or neither, since field 24 makes the + * row and its entries one record. + */ +void +test_txn_insert_tests_redo_from_any_point_gives_a_row_with_all_its_map_entries_or_none(void) +{ + static const struct + { + uint32 database; + size_t from; + } froms[] = {{9319, 0}, {9320, 2}, {9321, 3}}; + static const uint8 one[8] = {0, 0, 0, 0, 0, 0, 0, 1}; + static const StdFmtPiece pieces[] = {STD_FMT_LIT("from "), STD_FMT_ARG(0)}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld toast; + HostSharedHeld deltas; + TreesStoreReader store; + TxnValuesEntry entries[2]; + TxnInsertInsert row; + txn_insert_tests_mapped_t primary[2]; + uint64 tid; + StdVec writes; + + txn_insert_tests_repository(io, 9318, 2168, 87, 2169, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9318), 2170, 106); + toast = txn_insert_tests_toasted(io, &repo, &reading, 9318, 2179, 10); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + row = txn_insert_tests_valued(9318, entries); + deltas = txn_insert_tests_written(io, 9318, row.writer, entries); + host_shared_Held_drop(&deltas); + tid = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, 60, &toast, entries, 2, + 2184, 82); + txn_insert_tests_mapped(io, &toast, primary); + writes = txn_insert_tests_logged(9318); + STD_ASSERT_EQ_U64(2186, 9, writes.len, 3); + txn_insert_tests_logged_free(&writes); + for (size_t k = 0; k < lengthof(froms); k++) + { + StdFmtArg args[] = {{&froms[k].from, std_usize_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + txn_insert_tests_replica_t replica; + txn_insert_tests_from_t got = {0}; + txn_insert_tests_from_t want = {0}; + HostRow tuple = {tid}; + HostError error; + + writes = txn_insert_tests_logged(9318); + for (size_t w = 0; w < froms[k].from && w < writes.len; w++) + txn_insert_tests_logged_drop(&STD_VEC_AT(&writes, txn_insert_tests_logged_t, w)); + { + size_t skipped = Min(froms[k].from, writes.len); + + memmove(writes.ptr, &STD_VEC_AT(&writes, txn_insert_tests_logged_t, skipped), + (writes.len - skipped) * sizeof(txn_insert_tests_logged_t)); + writes.len -= skipped; + } + replica = txn_insert_tests_replica(io, froms[k].database, &writes); + if (!txn_memtable_tuples_get(io, host_shared_Held_record(&replica.held), tuple, &got.some, &got.row, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2190, 73, &error); + if (replica.has_toast) + txn_insert_tests_mapped(io, &replica.toast, got.values); + if (froms[k].from != 3) + { + want.some = true; + want.row = (StdVec) + { + (void *) one, sizeof(one), sizeof(one) + }; + want.values[0] = primary[0]; + want.values[1] = primary[1]; + } + if (!txn_insert_tests_from_eq(&got, &want) && froms[k].from == 3) + std_assert_failed(RUST_FILE, 2196, 22, "==", &got, txn_insert_tests_from_Debug_fmt, &want, + txn_insert_tests_from_Debug_fmt, &message); + if (!txn_insert_tests_from_eq(&got, &want)) + std_assert_failed(RUST_FILE, 2197, 22, "==", &got, txn_insert_tests_from_Debug_fmt, &want, + txn_insert_tests_from_Debug_fmt, &message); + for (size_t e = 0; e < 2; e++) + { + bool durable = txn_insert_tests_durable(io, &replica.deltas, &entries[e], 2200, 92); + bool wanted = froms[k].from == 0; + + STD_ASSERT_EQ(2200, 17, durable == wanted, &durable, std_bool_Debug_fmt, &wanted, + std_bool_Debug_fmt); + } + for (size_t e = 0; e < 2; e++) + txn_insert_tests_mapped_drop(&got.values[e]); + if (got.some) + std_vec_free_Checked(&got.row, 1, 1); + txn_insert_tests_replica_drop(&replica); + } + txn_insert_tests_ended(io, 2203, 35); + for (size_t k = 0; k < 2; k++) + txn_insert_tests_mapped_drop(&primary[k]); + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_counts reads the map counts of the test's two value ids in a TOAST relation, unwrapped at the lines and columns a case gives. */ +static void +txn_insert_tests_counts(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *toast, uint32 error_line, + uint32 error_col, uint32 none_line, uint32 none_col, uint64 counts[2]) +{ + for (size_t k = 0; k < 2; k++) + { + TxnValuesEntry entry; + HostError error; + bool has; + + if (!txn_values_entry(io, store, toast, TXN_INSERT_TESTS_IDS[k], &has, &entry, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(error_line, error_col, &error); + if (!has) + STD_OPTION_UNWRAP_FAILED(none_line, none_col); + counts[k] = entry.count; + } +} + +/* + * (4b)'s section 5: an update that keeps a row's two out-of-line values, + * field 28, redone adds one to each kept value's map entry with the record's + * LSN, and redone again, which the map holds by the id and that LSN, adds + * nothing. + */ +void +test_txn_insert_tests_a_kept_values_count_add_is_redone_once(void) +{ + static const uint64 once[] = {1, 1}; + static const uint64 twice[] = {2, 2}; + static const size_t kept[] = {0, 1}; + static const uint8 one[8] = {0, 0, 0, 0, 0, 0, 0, 1}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld toast; + HostSharedHeld deltas; + TreesStoreReader store; + TxnValuesEntry entries[2]; + TxnInsertInsert row; + bool has_relation; + TxnRecordToast relation; + uint64 tid; + TxnRecordRowChange update = {.writer = {.xid = 42}, .table = 16385, .heap_bytes = 60}; + ChunksAddressAddress overflow = chunks_address_Address_of((const uint8 *) "overflow", 8); + chunks_TupleU32Locator_t address = {.f0 = 0, .f1 = chunks_trailer_Locator_Default_default()}; + uint64 counts[2]; + HostError error; + + txn_insert_tests_repository(io, 9323, 2212, 87, 2213, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9323), 2214, 106); + toast = txn_insert_tests_toasted(io, &repo, &reading, 9323, 2223, 10); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + row = txn_insert_tests_valued(9323, entries); + deltas = txn_insert_tests_written(io, 9323, row.writer, entries); + host_shared_Held_drop(&deltas); + has_relation = row.has_toast; + relation = row.toast; + tid = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, 60, &toast, entries, 2, + 2229, 82); + update.storage = txn_insert_tests_stored(9323); + std_slice_to_vec_Checked(&update.key, one, sizeof(one), 1, 1); + update.op.tag = TXN_MEMTABLE_OP_PUT; + std_slice_to_vec_Checked(&update.op.u.put.value, overflow.f0, sizeof(overflow.f0), 1, 1); + std_slice_to_vec_Checked(&update.op.u.put.addresses, &address, 1, sizeof(chunks_TupleU32Locator_t), + _Alignof(chunks_TupleU32Locator_t)); + std_slice_to_vec_Checked(&update.op.u.put.values, TXN_INSERT_TESTS_IDS, 2, sizeof(uint32), _Alignof(uint32)); + update.op.u.put.tuple.f0 = tid + 1; + update.op.u.put.heap_bytes = 60; + update.has_toast = has_relation; + update.toast = relation; + std_vec_new(&update.blobs, sizeof(chunks_TupleAddressU64_t), _Alignof(chunks_TupleAddressU64_t)); + std_slice_to_vec_Checked(&update.kept, kept, lengthof(kept), sizeof(size_t), _Alignof(size_t)); + txn_insert_tests_counts(io, TREES_STORE_READER_NODE_STORE(&store), &toast, 2254, 82, 2254, 91, counts); + txn_insert_tests_u64s_is(2256, 9, counts, 2, once, 2); + if (!txn_insert_redo(io, TREES_STORE_READER_NODE_STORE(&store), &held, &update, 0x100000, &TXN_INSERT_TESTS_PAGE, + &toast, NULL, 0, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2257, 81, &error); + txn_insert_tests_counts(io, TREES_STORE_READER_NODE_STORE(&store), &toast, 2254, 82, 2254, 91, counts); + txn_insert_tests_u64s_is(2258, 9, counts, 2, twice, 2); + if (!txn_insert_redo(io, TREES_STORE_READER_NODE_STORE(&store), &held, &update, 0x100000, &TXN_INSERT_TESTS_PAGE, + &toast, NULL, 0, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2259, 81, &error); + txn_insert_tests_counts(io, TREES_STORE_READER_NODE_STORE(&store), &toast, 2254, 82, 2254, 91, counts); + txn_insert_tests_u64s_is(2260, 9, counts, 2, twice, 2); + txn_insert_tests_ended(io, 2261, 35); + txn_record_RowChange_drop(&update); + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_left_t is the load case's captures: the bytes of the free space entry a leave handed to the map. */ +typedef struct txn_insert_tests_left_t +{ + StdVec left; +} txn_insert_tests_left_t; + +/* txn_insert_tests_left_step is the load case's take's step: it puts the left page's entry. */ +static TxnMemtableMapStep +txn_insert_tests_left_step(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out) +{ + const txn_insert_tests_left_t *c = captures; + + std_vec_extend_from_slice_Checked(out, c->left.ptr, c->left.len, 1, 1); + return TXN_MEMTABLE_MAP_STEP_PUT; +} + +/* txn_insert_tests_left_then is the load case's take's then, which does nothing. */ +static void +txn_insert_tests_left_then(void *captures) +{ +} + +/* txn_insert_tests_left_inside is the load case's prepare's closure: block 0's take of the left page's entry, with no transaction. */ +static bool +txn_insert_tests_left_inside(void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, + TxnMemtableMapTaken *result, HostError *error) +{ + TxnStatusRoomed unroomed = {0}; + + return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, TXN_MEMTABLE_VERSION_NO_TRANSACTION, + false, unroomed, + false, 0, false, NULL, 0, NULL, txn_insert_tests_left_step, captures, NULL, + txn_insert_tests_left_then, NULL, NULL, result, error); +} + +/* txn_insert_tests_loaded_t is the load case's state: the counter and the first four blocks' free space entries. */ +typedef txn_insert_tests_free_t txn_insert_tests_loaded_t; + +/* txn_insert_tests_loaded_Debug_fmt is the load case's state's Debug, the tuple (counter, entries). */ +static bool +txn_insert_tests_loaded_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_free_t *loaded = value; + StdVec entries = {(void *) loaded->entries, lengthof(loaded->entries), lengthof(loaded->entries)}; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&loaded->counter, std_u64_Debug_fmt), + STD_ITEM(&entries, txn_insert_tests_mappeds_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_insert_tests_load runs the runs' loader on a storage up to a tuple id, unwrapped at a line and column. */ +static void +txn_insert_tests_load(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint64 to, uint32 line, + uint32 col) +{ + HostError error; + + if (!txn_insert_loaded(io, store, held, &TXN_INSERT_TESTS_PAGE, to, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(line, col, &error); +} + +/* + * The checkpoint runs text's L2: a restart whose loaded runs hold rows up to + * (1,5), and block 0's free space entry from a page a leave handed to the + * map, offsets 3 on, moves the counter to (1,6), the tuple id its loader + * gives, and leaves the entry as loaded; a second load changes nothing; an + * unrelated transaction's abort leaves the move and the entry; and the next + * inserts take (0,3) and (0,4), never a loaded row's tuple id. + */ +void +test_txn_insert_tests_a_load_moves_the_counter_past_every_loaded_tid_and_left_page(void) +{ + static const uint64 want[] = {3, 4}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + txn_insert_tests_left_t c; + uint16 offsets[289]; + TxnFreeSpaceFree free; + TxnMemtableMapTake take; + TxnMemtableMapTaken taken; + txn_insert_tests_free_t expected; + txn_insert_tests_free_t again; + HostXid unrelated; + HostError error; + uint64 tids[2]; + bool eq; + + txn_insert_tests_repository(io, 9322, 2273, 87, 2274, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9322), 2275, 106); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (uint16 offset = 3; offset <= 291; offset++) + offsets[offset - 3] = offset; + free = txn_free_space_Free_new(8168 - 6000, offsets, lengthof(offsets), 291); + std_vec_new(&c.left, 1, 1); + txn_free_space_Free_encode_into(&free, &c.left); + take.state = host_shared_Held_record(&held); + take.map = TXN_MEMTABLE_MAP_MAP_FREE; + take.key = 0; + take.most = txn_free_space_most_len(291); + if (!txn_memtable_map_prepare_Taken(io, &take, 1, txn_insert_tests_left_inside, &c, NULL, &taken, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2287, 10, &error); + txn_insert_tests_load(io, TREES_STORE_READER_NODE_STORE(&store), &held, 1 << 16 | 6, 2290, 55); + expected = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + STD_ASSERT_EQ_U64(2292, 9, expected.counter, 1 << 16 | 6); + { + txn_insert_tests_mapped_t left = {.some = true, .entry = {.has_value = true, .value = c.left}}; + + STD_ASSERT_EQ(2293, 9, txn_insert_tests_mapped_eq(&expected.entries[0], &left), &expected.entries[0], + txn_insert_tests_mapped_Debug_fmt, &left, txn_insert_tests_mapped_Debug_fmt); + } + txn_insert_tests_load(io, TREES_STORE_READER_NODE_STORE(&store), &held, 1 << 16 | 6, 2294, 55); + unrelated = simc_begin(); + simc_abort(unrelated); + again = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + eq = again.counter == expected.counter; + for (size_t k = 0; eq && k < lengthof(again.entries); k++) + eq = txn_insert_tests_mapped_eq(&again.entries[k], &expected.entries[k]); + STD_ASSERT_EQ(2297, 9, eq, &again, txn_insert_tests_loaded_Debug_fmt, &expected, + txn_insert_tests_loaded_Debug_fmt); + for (uint64 n = 10; n < 12; n++) + tids[n - 10] = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9322), n), 28, NULL, NULL, 0, + 2300, 90); + txn_insert_tests_u64s_is(2301, 9, tids, 2, want, 2); + txn_insert_tests_ended(io, 2302, 35); + txn_insert_tests_free_drop(&again); + txn_insert_tests_free_drop(&expected); + std_vec_free_Checked(&c.left, 1, 1); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_ours returns the new cid records the simulation's WAL took for database 9324's storages, in their order. */ +static StdVec +txn_insert_tests_ours(void) +{ + StdVec cids = simc_new_cids(); + StdVec ours; + + std_vec_new(&ours, sizeof(simc_TupleStorageRowNewCidLsn_t), _Alignof(simc_TupleStorageRowNewCidLsn_t)); + for (size_t k = 0; k < cids.len; k++) + { + const simc_TupleStorageRowNewCidLsn_t *cid = &STD_VEC_AT(&cids, simc_TupleStorageRowNewCidLsn_t, k); + + if (cid->f0.database == 9324) + std_vec_push_Checked(&ours, cid, sizeof(*cid), _Alignof(simc_TupleStorageRowNewCidLsn_t)); + } + std_vec_free_Checked(&cids, sizeof(simc_TupleStorageRowNewCidLsn_t), _Alignof(simc_TupleStorageRowNewCidLsn_t)); + return ours; +} + +/* txn_insert_tests_own returns the end of the newest row change database 9324's storages logged, if any. */ +static bool +txn_insert_tests_own(HostLsn *end) +{ + StdVec writes = txn_insert_tests_logged(9324); + bool found = false; + + for (size_t k = writes.len; !found && k > 0; k--) + { + const txn_insert_tests_logged_t *write = &STD_VEC_AT(&writes, txn_insert_tests_logged_t, k - 1); + + if (write->tag == TXN_INSERT_TESTS_LOGGED_ROW) + { + *end = write->end; + found = true; + } + } + txn_insert_tests_logged_free(&writes); + return found; +} + +/* + * CommPF 5's NEW_CID ruling: a logged insert into a relation logical + * decoding reads logs heap's record of its row's command before its own + * record, with the row's tuple id and the inserting command, as heap_insert + * does; one into any other relation, or an unlogged one, logs none. + */ +void +test_txn_insert_tests_an_insert_logical_decoding_reads_logs_heaps_record_of_its_command(void) +{ + static const struct + { + bool logged; + bool decoded; + } cases[] = {{true, false}, {false, true}, {true, true}}; + static const StdFmtPiece pieces[] = {STD_FMT_LIT("logged "), STD_FMT_ARG(0), STD_FMT_LIT(", decoded "), + STD_FMT_ARG(1)}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + + txn_insert_tests_repository(io, 9324, 2312, 87, 2313, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9324), 2314, 106); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (size_t n = 0; n < lengthof(cases); n++) + { + bool logged = cases[n].logged; + bool decoded = cases[n].decoded; + TxnInsertInsert inserted = txn_insert_tests_row(txn_insert_tests_stored(9324), n); + uint64 tid; + StdVec cids; + + inserted.logged = logged; + inserted.decoded = decoded; + inserted.writer.command = 5; + tid = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, inserted, 28, NULL, NULL, 0, + 2328, 71); + cids = txn_insert_tests_ours(); + if (cids.len == 1 && logged && decoded) + { + const simc_TupleStorageRowNewCidLsn_t *cid = &STD_VEC_AT(&cids, simc_TupleStorageRowNewCidLsn_t, 0); + HostStorage storage = txn_insert_tests_stored(9324); + HostRow row = {tid}; + HostNewCid want = {.has_cmin = true, .cmin = {5}}; + StdTupleRef got = STD_TUPLE(3, STD_ITEM(&cid->f0, host_Storage_Debug_fmt), + STD_ITEM(&cid->f1, host_Row_Debug_fmt), + STD_ITEM(&cid->f2, host_NewCid_Debug_fmt)); + StdTupleRef wanted = STD_TUPLE(3, STD_ITEM(&storage, host_Storage_Debug_fmt), + STD_ITEM(&row, host_Row_Debug_fmt), + STD_ITEM(&want, host_NewCid_Debug_fmt)); + HostLsn own; + + STD_ASSERT_EQ(2333, 21, host_Storage_PartialEq_eq(&cid->f0, &storage) && cid->f1.f0 == tid && + cid->f2.has_cmin && cid->f2.cmin.f0 == 5 && !cid->f2.has_cmax && !cid->f2.has_combo, + &got, std_tuple_Debug_fmt, &wanted, std_tuple_Debug_fmt); + STD_ASSERT(2334, 21, txn_insert_tests_own(&own) && cid->f3 < own, + "own().is_some_and(|own| end < own)"); + } + else if (!(cids.len == 0 && !(logged && decoded))) + { + StdFmtArg args[] = {{&logged, std_bool_Display_fmt}, {&decoded, std_bool_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_panic_fmt(RUST_FILE, 2336, 22, &message); + } + std_vec_free_Checked(&cids, sizeof(simc_TupleStorageRowNewCidLsn_t), + _Alignof(simc_TupleStorageRowNewCidLsn_t)); + } + txn_insert_tests_ended(io, 2339, 35); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_chain_t is a key of a storage's rows with its count of versions, as a walk hands them out. */ +typedef struct txn_insert_tests_chain_t +{ + StdVec key; + size_t versions; +} txn_insert_tests_chain_t; + +/* txn_insert_tests_chains_each is chains' step: a key and its count of versions pushed onto the list. */ +static bool +txn_insert_tests_chains_each(void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *values, + size_t values_len, HostError *error) +{ + txn_insert_tests_chain_t chain = {.versions = values_len}; + + std_slice_to_vec_Checked(&chain.key, key, key_len, 1, 1); + std_vec_push_Checked(captures, &chain, sizeof(chain), _Alignof(txn_insert_tests_chain_t)); + return true; +} + +/* txn_insert_tests_chains returns each key of a storage's rows with its count of versions, as a walk hands them out. */ +static StdVec +txn_insert_tests_chains(HostIoIo *io, HostSharedRecord record) +{ + StdVec chains; + HostError error; + + std_vec_new(&chains, sizeof(txn_insert_tests_chain_t), _Alignof(txn_insert_tests_chain_t)); + if (!txn_memtable_table_walked(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, record, txn_insert_tests_chains_each, + &chains, NULL, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2349, 56, &error); + return chains; +} + +/* txn_insert_tests_chains_free frees a list of a storage's keys with their counts. */ +static void +txn_insert_tests_chains_free(StdVec *chains) +{ + for (size_t k = 0; k < chains->len; k++) + std_vec_free_Checked(&STD_VEC_AT(chains, txn_insert_tests_chain_t, k).key, 1, 1); + std_vec_free_Checked(chains, sizeof(txn_insert_tests_chain_t), _Alignof(txn_insert_tests_chain_t)); +} + +/* txn_insert_tests_chain_Debug_fmt is a key and its count's Debug, the tuple (key, count). */ +static bool +txn_insert_tests_chain_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_chain_t *chain = value; + StdBytes key = {chain->key.ptr, chain->key.len}; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&key, std_bytes_Debug_fmt), + STD_ITEM(&chain->versions, std_usize_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_insert_tests_chains_Debug_fmt is a Vec<(Vec, usize)>'s Debug. */ +static bool +txn_insert_tests_chains_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *chains = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < chains->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(chains, txn_insert_tests_chain_t, k), txn_insert_tests_chain_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_insert_tests_chains_eq reports whether two lists of a storage's keys with their counts are equal, as Vec<(Vec, usize)>s compare. */ +static bool +txn_insert_tests_chains_eq(const StdVec *a, const StdVec *b) +{ + bool eq = a->len == b->len; + + for (size_t k = 0; eq && k < a->len; k++) + { + const txn_insert_tests_chain_t *left = &STD_VEC_AT(a, txn_insert_tests_chain_t, k); + const txn_insert_tests_chain_t *right = &STD_VEC_AT(b, txn_insert_tests_chain_t, k); + + eq = left->versions == right->versions && trees_store_VecU8_PartialEq_eq(&left->key, &right->key); + } + return eq; +} + +/* + * txn_insert_tests_held_once asserts at a line and column, with a message or + * none, that a storage's rows are keys 0 up to a count, each with one + * version, and frees the list. + */ +static void +txn_insert_tests_held_once(uint32 line, uint32 col, StdVec *chains, uint64 count, const StdFmtArguments *message) +{ + StdVec want; + + std_vec_new(&want, sizeof(txn_insert_tests_chain_t), _Alignof(txn_insert_tests_chain_t)); + for (uint64 n = 0; n < count; n++) + { + txn_insert_tests_chain_t chain = {.versions = 1}; + uint8 key[8]; + + std_u64_to_be_bytes(n, key); + std_slice_to_vec_Checked(&chain.key, key, sizeof(key), 1, 1); + std_vec_push_Checked(&want, &chain, sizeof(chain), _Alignof(txn_insert_tests_chain_t)); + } + if (!txn_insert_tests_chains_eq(chains, &want)) + std_assert_failed(RUST_FILE, line, col, "==", chains, txn_insert_tests_chains_Debug_fmt, &want, + txn_insert_tests_chains_Debug_fmt, message); + txn_insert_tests_chains_free(&want); + txn_insert_tests_chains_free(chains); +} + +/* txn_insert_tests_no_edits is the checkpoint case's save's gathering, which edits nothing. */ +static bool +txn_insert_tests_no_edits(void *captures, HostIoIo *io, const RepoRead *read, StdVec *result, ChunksErrorError *error) +{ + std_vec_new(result, sizeof(RepoGathered), _Alignof(RepoGathered)); + return true; +} + +/* txn_insert_tests_no_marks is the checkpoint case's save's marks, which mark nothing. */ +static bool +txn_insert_tests_no_marks(void *captures, HostIoIo *io, ChunksErrorError *error) +{ + return true; +} + +/* + * I12 and the checkpoint runs text's straddle case: a checkpoint's runs + * loaded after a crash and followed by every logged record leave each + * block's free space entry and the counter as the primary's, so no tid is + * handed out twice, and every row held once: when the runs hold everything, + * so redo pushes no row again; when they hold the inserts' free space takes + * but not their rows; and when they are a restartpoint's, written in the + * middle of the records' redo. + */ +void +test_txn_insert_tests_a_checkpoints_runs_and_every_record_after_redo_as_the_primary(void) +{ + static const struct + { + uint32 source; + uint32 database; + bool rows; + size_t redone_before; + } cases[] = {{9326, 9327, true, SIZE_MAX}, {9328, 9329, false, SIZE_MAX}, {9331, 9332, true, 5}}; + static const uint64 sizes[] = {3000, 3000, 3000, 3000, 3000, 2000, 3000, 2000}; + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + + for (size_t k = 0; k < lengthof(cases); k++) + { + uint32 database = cases[k].database; + StdFmtArg args[] = {{&database, std_u32_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + RepoRepo primary; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld deltas; + TreesStoreReader store; + txn_insert_tests_free_t expected; + txn_insert_tests_free_t got; + txn_insert_tests_replica_t replica; + StdArcInner *manifest; + const RepoManifest *newest; + RepoRuns runs; + StdVec before; + StdVec writes; + StdVec chains; + ChunksErrorError failed; + HostError error; + size_t held_before; + size_t kept = 0; + + txn_insert_tests_repository(io, cases[k].source, 2367, 96, 2368, 50, &primary, &reading); + held = txn_insert_tests_stated(io, &primary, &reading, txn_insert_tests_stored(cases[k].source), + 2370, 108); + store = repo_Repo_reader(&primary, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (uint64 n = 0; n < lengthof(sizes); n++) + txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(cases[k].source), n), sizes[n], NULL, + NULL, 0, 2374, 79); + expected = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + before = txn_insert_tests_logged(cases[k].source); + for (size_t w = Min(cases[k].redone_before, before.len); w < before.len; w++) + txn_insert_tests_logged_drop(&STD_VEC_AT(&before, txn_insert_tests_logged_t, w)); + before.len = Min(cases[k].redone_before, before.len); + held_before = txn_insert_tests_count(&before, TXN_INSERT_TESTS_LOGGED_ROW); + replica = txn_insert_tests_replica(io, database, &before); + txn_insert_tests_replica_drop(&replica); + host_shared_Held_drop(&held); + simc_checkpointed(UINT64CONST(0x100000000)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000001), &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2383, 56, &error); + if (!repo_Repo_open(io, database, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(2384, 61, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!repo_Repo_manifest(&repo, io, &manifest, &failed)) + STD_RESULT_EXPECT_FAILED(2385, 46, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + newest = repo_Repo_newest(manifest); + if (!newest->has_runs) + STD_OPTION_UNWRAP_FAILED(2385, 68); + repo_Runs_Clone_clone(&newest->runs, &runs); + repo_ArcManifest_drop(manifest); + for (size_t f = 0; f < runs.files.len; f++) + { + const RepoRunFile *file = &STD_VEC_AT(&runs.files, RepoRunFile, f); + + if (cases[k].rows || file->kind != TXN_RUNS_KIND_ROWS) + STD_VEC_AT(&runs.files, RepoRunFile, kept++) = *file; + } + runs.files.len = kept; + if (!repo_Repo_save(&repo, io, &reading, true, &runs, txn_insert_tests_no_edits, NULL, + txn_insert_tests_no_marks, NULL, &failed)) + STD_RESULT_EXPECT_FAILED(2387, 84, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + for (size_t u = 0; u < 2; u++) + { + HostStorage unlinked = txn_insert_tests_stored(u == 0 ? cases[k].source : database); + + if (!txn_state_unlinked(io, &unlinked, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2389, 56, &error); + } + if (!txn_runs_loaded(io, &TXN_INSERT_TESTS_PAGE, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2391, 71, &error); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(database), 2393, 107); + chains = txn_insert_tests_chains(io, host_shared_Held_record(&held)); + { + size_t len = chains.len; + size_t want = cases[k].rows ? held_before : 0; + + if (len != want) + std_assert_failed(RUST_FILE, 2394, 13, "==", &len, std_usize_Debug_fmt, &want, + std_usize_Debug_fmt, &message); + } + txn_insert_tests_chains_free(&chains); + if (!txn_memtable_branch_deltas(io, database, HOST_PG_DEFAULT, &deltas, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2395, 69, &error); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + writes = txn_insert_tests_logged(cases[k].source); + txn_insert_tests_redone(io, TREES_STORE_READER_NODE_STORE(&store), &held, NULL, &deltas, &writes); + got = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + txn_insert_tests_free_is(2398, 13, &got, &expected, &message); + txn_insert_tests_free_drop(&got); + chains = txn_insert_tests_chains(io, host_shared_Held_record(&held)); + txn_insert_tests_held_once(2400, 13, &chains, 8, &message); + writes = txn_insert_tests_logged(cases[k].source); + for (size_t w = 0; w < writes.len; w++) + { + const txn_insert_tests_logged_t *write = &STD_VEC_AT(&writes, txn_insert_tests_logged_t, w); + const TxnMemtablePut *put; + StdVec tupled; + bool some; + + if (write->tag != TXN_INSERT_TESTS_LOGGED_ROW || !txn_record_RowChange_put(&write->u.row, &put)) + continue; + if (!txn_memtable_tuples_get(io, host_shared_Held_record(&held), put->tuple, &some, &tupled, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2405, 74, &error); + if (!some || !trees_store_VecU8_PartialEq_eq(&tupled, &write->u.row.key)) + { + StdBytes row = {some ? tupled.ptr : NULL, some ? tupled.len : 0}; + StdBytes key = {write->u.row.key.ptr, write->u.row.key.len}; + StdOptionRef left = {some, {&row, std_bytes_Debug_fmt}}; + StdOptionRef right = {true, {&key, std_bytes_Debug_fmt}}; + + std_assert_failed(RUST_FILE, 2405, 21, "==", &left, std_option_Debug_fmt, &right, + std_option_Debug_fmt, &message); + } + std_vec_free_Checked(&tupled, 1, 1); + } + txn_insert_tests_logged_free(&writes); + host_shared_Held_drop(&deltas); + host_shared_Held_drop(&held); + repo_Repo_drop(&repo); + txn_insert_tests_free_drop(&expected); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&primary); + } + txn_insert_tests_ended(io, 2409, 35); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* txn_insert_tests_straddle_t is the straddle's inserter's captures: its database, its count of inserts, and its rows' heap bytes. */ +typedef struct txn_insert_tests_straddle_t +{ + uint32 database; + uint64 inserts; + uint64 bytes; +} txn_insert_tests_straddle_t; + +/* txn_insert_tests_checkpointer is a case's checkpointer: a checkpoint whose redo is past the test's last. */ +static bool +txn_insert_tests_checkpointer(void *captures, HostIoIo *io, HostError *error) +{ + return txn_runs_checkpoint(io, UINT64CONST(0x100000001), error); +} + +/* txn_insert_tests_straddle_inserter is the straddle's inserter: its inserts, one window each, while the checkpoint waits. */ +static bool +txn_insert_tests_straddle_inserter(void *captures, HostIoIo *io, HostError *error) +{ + const txn_insert_tests_straddle_t *c = captures; + HostStorage storage = txn_insert_tests_stored(c->database); + TxnInsertRoom room = txn_insert_tests_room(c->bytes); + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + bool ok = true; + + if (!txn_insert_tests_opened(io, storage, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + for (uint64 n = 1; ok && n <= c->inserts; n++) + { + TxnInsertInsert row = txn_insert_tests_row(storage, n); + uint64 tid; + + ok = txn_insert_tests_taken(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &room, NULL, NULL, 0, + &tid, error); + } + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * txn_insert_tests_straddled runs a count of inserts of rows of some heap + * bytes, each a whole window, while a checkpoint waits at a pause of its sweep + * of their storage, which holds one such row already, then crashes, loads the + * checkpoint's runs, which hold that one row, and redoes the logged records, + * and checks every block's free space entry and the counter are as they were + * before the crash and each row is held once. + */ +static void +txn_insert_tests_straddled(uint32 database, const char *pause, uint64 bytes, uint64 inserts) +{ + static const SimcStep steps[] = {{"inserter", 8, UINT64_MAX}, {"checkpointer", 12, UINT64_MAX}}; + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostStorage storage = txn_insert_tests_stored(database); + txn_insert_tests_straddle_t c = {.database = database, .inserts = inserts, .bytes = bytes}; + SimcActor checkpointer = {txn_insert_tests_checkpointer}; + SimcActor inserter = {txn_insert_tests_straddle_inserter, &c}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld deltas; + TreesStoreReader store; + txn_insert_tests_free_t expected; + txn_insert_tests_free_t got; + SimcRun run; + SimcReport report; + StdVec writes; + HostError error; + + txn_insert_tests_repository(io, database, 2422, 59, 2423, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, storage, 2426, 99); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, txn_insert_tests_row(storage, 0), bytes, + NULL, NULL, 0, 2428, 62); + host_shared_Held_drop(&held); + simc_checkpointed(UINT64CONST(0x100000000)); + run = simc_Run_new(1); + simc_Run_actor(&run, "checkpointer", 12, checkpointer); + simc_Run_actor(&run, "inserter", 8, inserter); + simc_Run_until(&run, "checkpointer", 12, pause); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, RUST_FILE, 2442, 67, 2443, 9); + simc_Report_drop(&report); + held = txn_insert_tests_stated(io, &repo, &reading, storage, 2446, 99); + expected = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + host_shared_Held_drop(&held); + if (!txn_state_unlinked(io, &storage, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2449, 33, &error); + if (!txn_runs_loaded(io, &TXN_INSERT_TESTS_PAGE, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2450, 67, &error); + held = txn_insert_tests_stated(io, &repo, &reading, storage, 2451, 95); + { + StdVec chains = txn_insert_tests_chains(io, host_shared_Held_record(&held)); + + txn_insert_tests_held_once(2453, 9, &chains, 1, NULL); + } + if (!txn_memtable_branch_deltas(io, database, HOST_PG_DEFAULT, &deltas, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2454, 65, &error); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + writes = txn_insert_tests_logged(database); + txn_insert_tests_redone(io, TREES_STORE_READER_NODE_STORE(&store), &held, NULL, &deltas, &writes); + got = txn_insert_tests_free_entries(io, host_shared_Held_record(&held)); + txn_insert_tests_free_is(2457, 9, &got, &expected, NULL); + txn_insert_tests_free_drop(&got); + { + StdVec chains = txn_insert_tests_chains(io, host_shared_Held_record(&held)); + + txn_insert_tests_held_once(2458, 9, &chains, 1 + inserts, NULL); + } + txn_insert_tests_ended(io, 2459, 35); + host_shared_Held_drop(&deltas); + host_shared_Held_drop(&held); + txn_insert_tests_free_drop(&expected); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * The sweep's order: an insert whose whole window runs between a + * checkpoint's sweep of its storage's rows and its cut of the free space map + * leaves the runs its take without its row, under the cut's counter, so redo + * pushes the row and takes nothing twice. + */ +void +test_txn_insert_tests_a_window_between_a_sweeps_rows_and_its_cut_leaves_its_take_without_its_row(void) +{ + txn_insert_tests_straddled(9330, "doltgres-runs-rows-swept", 28, 1); +} + +/* + * The cut: inserts whose whole windows run after a checkpoint's cut of the + * free space map, before the cut's walk, are in the runs with neither their + * rows nor their takes, whatever the walk meets after of the counter's block + * and of the block the inserts go on to once they leave it, so redo applies + * each once. + */ +void +test_txn_insert_tests_windows_after_a_sweeps_cut_are_in_its_runs_with_neither_their_rows_nor_their_takes(void) +{ + txn_insert_tests_straddled(9333, "doltgres-runs-free-cut", 3000, 3); +} + +/* txn_insert_tests_value_entry returns a value's map entry of 4000 bytes, counted once, by its bytes' address. */ +static TxnValuesEntry +txn_insert_tests_value_entry(const char *bytes) +{ + TxnValuesEntry entry = {.raw = 4000, .stored = 4000, .count = 1}; + + entry.address = chunks_address_Address_of((const uint8 *) bytes, strlen(bytes)); + entry.locator = chunks_trailer_Locator_Default_default(); + return entry; +} + +/* txn_insert_tests_valued_by returns a key's row written by a transaction, naming one value id in the test's TOAST relation. */ +static TxnInsertInsert +txn_insert_tests_valued_by(uint64 key, HostXid xid, uint32 id) +{ + TxnInsertInsert row = txn_insert_tests_row(txn_insert_tests_stored(9388), key); + + row.writer.xid = xid.f0; + std_vec_free_Checked(&row.put.values, sizeof(uint32), _Alignof(uint32)); + std_slice_to_vec_Checked(&row.put.values, &id, 1, sizeof(uint32), _Alignof(uint32)); + row.has_toast = true; + row.toast = TXN_INSERT_TESTS_TOAST; + return row; +} + +/* + * Bugs row 44, CommPF 7's condition on 2g's take-back: an aborted insert's + * value map entries go with its row at its take-back, since its list holds + * their keys, and a committed insert's stay; its free space take stays, for + * the fold or VACUUM to free as heap's VACUUM frees a dead row's tuple id. + */ +void +test_txn_insert_tests_an_aborted_inserts_value_map_versions_go_at_its_take_back_and_its_free_space_take_stays(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld toast; + TreesStoreReader store; + TxnValuesEntry one = txn_insert_tests_value_entry("one"); + TxnValuesEntry two = txn_insert_tests_value_entry("two"); + HostXid committed; + HostXid aborted; + txn_insert_tests_mapped_t mapped; + txn_insert_tests_mapped_t free; + txn_insert_tests_mapped_t now; + txn_insert_tests_mapped_t none = {0}; + HostDeltaHold hold; + HostError error; + StdVec chain; + uint8 key[8]; + + txn_insert_tests_repository(io, 9388, 2486, 87, 2487, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9388), 2488, 106); + toast = txn_insert_tests_toasted(io, &repo, &reading, 9388, 2490, 102); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + committed = simc_begin(); + aborted = simc_begin(); + txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_valued_by(1, committed, 16400), 60, &toast, &one, 1, 2505, 82); + simc_commit(committed); + txn_insert_tests_ended(io, 2507, 28); + txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_valued_by(2, aborted, 16401), 60, &toast, &two, 1, 2512, 83); + mapped = txn_insert_tests_newest(io, host_shared_Held_record(&toast), TXN_MEMTABLE_MAP_MAP_VALUES, 16401, + 2513, 69); + STD_ASSERT(2513, 9, mapped.some, + "map::newest(io, toast.record(), Map::Values, 16401).unwrap().is_some()"); + txn_insert_tests_mapped_drop(&mapped); + free = txn_insert_tests_newest(io, host_shared_Held_record(&held), TXN_MEMTABLE_MAP_MAP_FREE, 0, 2514, 65); + simc_abort(aborted); + if (!txn_written_aborted(io, 0, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2516, 30, &error); + mapped = txn_insert_tests_newest(io, host_shared_Held_record(&toast), TXN_MEMTABLE_MAP_MAP_VALUES, 16401, + 2517, 72); + STD_ASSERT_EQ(2517, 9, txn_insert_tests_mapped_eq(&mapped, &none), &mapped, + txn_insert_tests_mapped_Debug_fmt, &none, txn_insert_tests_mapped_Debug_fmt); + txn_insert_tests_mapped_drop(&mapped); + mapped = txn_insert_tests_newest(io, host_shared_Held_record(&toast), TXN_MEMTABLE_MAP_MAP_VALUES, 16400, + 2518, 69); + STD_ASSERT(2518, 9, mapped.some, + "map::newest(io, toast.record(), Map::Values, 16400).unwrap().is_some()"); + txn_insert_tests_mapped_drop(&mapped); + now = txn_insert_tests_newest(io, host_shared_Held_record(&held), TXN_MEMTABLE_MAP_MAP_FREE, 0, 2519, 65); + STD_ASSERT_EQ(2519, 9, txn_insert_tests_mapped_eq(&now, &free), &now, txn_insert_tests_mapped_Debug_fmt, + &free, txn_insert_tests_mapped_Debug_fmt); + txn_insert_tests_mapped_drop(&now); + if (!host_delta_hold(io, &hold, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2520, 42, &error); + std_u64_to_be_bytes(2, key); + if (!txn_memtable_table_versions(io, &hold, host_shared_Held_record(&held), key, sizeof(key), &chain, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2521, 80, &error); + STD_ASSERT(2521, 9, chain.len == 0, + "table::versions(io, &hold, held.record(), &2u64.to_be_bytes()).unwrap().is_empty()"); + txn_memtable_version_versions_drop(&chain, 0); + std_u64_to_be_bytes(1, key); + if (!txn_memtable_table_versions(io, &hold, host_shared_Held_record(&held), key, sizeof(key), &chain, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2522, 83, &error); + STD_ASSERT_EQ_U64(2522, 9, chain.len, 1); + txn_memtable_version_versions_drop(&chain, 0); + host_delta_Hold_drop(&hold); + txn_insert_tests_mapped_drop(&free); + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_op_t is what a version of a key does, by its name, with the tuple ids it names or writes. */ +typedef struct txn_insert_tests_op_t +{ + const char *name; + uint64 at; +} txn_insert_tests_op_t; + +/* txn_insert_tests_named returns the tuple id at the end of a successor's key, or 0 for any other successor. */ +static uint64 +txn_insert_tests_named(bool has, const TxnMemtableVersionSuccessorRef *successor) +{ + if (has && successor->tag == TXN_MEMTABLE_VERSION_SUCCESSOR_REF_KEY && successor->u.key.len > 8) + return std_u64_from_be_bytes(successor->u.key.ptr + successor->u.key.len - 8); + return 0; +} + +/* txn_insert_tests_ops returns what each version of a key does, newest first, with the tuple ids it names or writes. */ +static StdVec +txn_insert_tests_ops(HostIoIo *io, HostSharedRecord record, const uint8 *key, size_t key_len) +{ + HostDeltaHold hold; + HostError error; + StdVec versions; + StdVec ops; + + if (!host_delta_hold(io, &hold, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2528, 42, &error); + if (!txn_memtable_table_versions(io, &hold, record, key, key_len, &versions, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2529, 64, &error); + std_vec_with_capacity_Checked(&ops, versions.len, sizeof(txn_insert_tests_op_t), _Alignof(txn_insert_tests_op_t)); + for (size_t k = 0; k < versions.len; k++) + { + TxnMemtableVersionRecord got = txn_memtable_version_Version_record(&STD_VEC_AT(&versions, + TxnMemtableVersionVersion, k)); + TxnMemtableVersionOpRef op = txn_memtable_version_Record_op(&got); + txn_insert_tests_op_t one = {"other", 0}; + + switch (op.tag) + { + case TXN_MEMTABLE_VERSION_OP_REF_PUT: + one = (txn_insert_tests_op_t) + { + "put", op.u.put.tuple.f0 + }; + break; + case TXN_MEMTABLE_VERSION_OP_REF_ADD: + one = (txn_insert_tests_op_t) + { + "add", op.u.add.tuple.f0 + }; + break; + case TXN_MEMTABLE_VERSION_OP_REF_ADD_ROW: + one = (txn_insert_tests_op_t) + { + "add row", op.u.add_row.tuple.f0 + }; + break; + case TXN_MEMTABLE_VERSION_OP_REF_UPDATE_ROW: + one = (txn_insert_tests_op_t) + { + "update row", op.u.update_row.f0.f0 << 32 | op.u.update_row.f1.tuple.f0 + }; + break; + case TXN_MEMTABLE_VERSION_OP_REF_MOVE: + one = (txn_insert_tests_op_t) + { + "move", 0 + }; + break; + case TXN_MEMTABLE_VERSION_OP_REF_REMOVE: + one = (txn_insert_tests_op_t) + { + "remove", op.u.remove.f0.f0 << 32 | + txn_insert_tests_named(op.u.remove.has_f1, &op.u.remove.f1) + }; + break; + case TXN_MEMTABLE_VERSION_OP_REF_DELETE_ROW: + one = (txn_insert_tests_op_t) + { + "delete row", op.u.delete_row.f0.f0 << 32 | + txn_insert_tests_named(op.u.delete_row.has_f1, &op.u.delete_row.f1) + }; + break; + default: + break; + } + std_vec_push_Checked(&ops, &one, sizeof(one), _Alignof(txn_insert_tests_op_t)); + } + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + return ops; +} + +/* txn_insert_tests_op_Debug_fmt is what a version does's Debug, the tuple (name, tuple id). */ +static bool +txn_insert_tests_op_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_op_t *op = value; + StdStr name = {op->name, strlen(op->name)}; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&name, std_str_Debug_fmt), STD_ITEM(&op->at, std_u64_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_insert_tests_ops_Debug_fmt is a Vec<(&str, u64)>'s Debug. */ +static bool +txn_insert_tests_ops_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *ops = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < ops->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(ops, txn_insert_tests_op_t, k), txn_insert_tests_op_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_insert_tests_ops_is asserts at a line and column that a key's versions do what another list says, and frees both lists. */ +static void +txn_insert_tests_ops_is(uint32 line, uint32 col, StdVec *got, StdVec *want) +{ + bool eq = got->len == want->len; + + for (size_t k = 0; eq && k < got->len; k++) + { + const txn_insert_tests_op_t *a = &STD_VEC_AT(got, txn_insert_tests_op_t, k); + const txn_insert_tests_op_t *b = &STD_VEC_AT(want, txn_insert_tests_op_t, k); + + eq = a->at == b->at && strcmp(a->name, b->name) == 0; + } + STD_ASSERT_EQ(line, col, eq, got, txn_insert_tests_ops_Debug_fmt, want, txn_insert_tests_ops_Debug_fmt); + std_vec_free_Checked(got, sizeof(txn_insert_tests_op_t), _Alignof(txn_insert_tests_op_t)); + std_vec_free_Checked(want, sizeof(txn_insert_tests_op_t), _Alignof(txn_insert_tests_op_t)); +} + +/* txn_insert_tests_ops_are asserts at a line and column that a key's versions do what an array says, and frees the list. */ +static void +txn_insert_tests_ops_are(uint32 line, uint32 col, StdVec got, const txn_insert_tests_op_t *want, size_t want_len) +{ + StdVec wanted; + + std_slice_to_vec_Checked(&wanted, want, want_len, sizeof(txn_insert_tests_op_t), _Alignof(txn_insert_tests_op_t)); + txn_insert_tests_ops_is(line, col, &got, &wanted); +} + +/* txn_insert_tests_updated_decide is updated's read: on with the command go shows, over the newest, or with none a stop with the count of versions it read. */ +static bool +txn_insert_tests_updated_decide(void *captures, HostIoIo *io, TxnMemtableVersionVersion *versions, size_t versions_len, + TxnChangeRead *result, void *stopped, HostError *error) +{ + const TxnMemtableVersionShown *go = captures; + + if (go == NULL) + { + *(size_t *) stopped = versions_len; + result->tag = TXN_CHANGE_READ_STOP; + return true; + } + result->tag = TXN_CHANGE_READ_GO; + result->u.go.shown = *go; + result->u.go.cmin = 0; + result->u.go.has_row = versions_len > 0; + if (versions_len > 0) + result->u.go.row = versions[0].handle; + return true; +} + +/* + * txn_insert_tests_updated runs an update of a test's row with a test's + * Waits, each key's stripe by its bytes, and a read that goes on with the + * command go shows, or with none stops with the count of versions it read, + * which it writes to stopped. It takes the row and the replaced key. + */ +static TxnInsertInserted +txn_insert_tests_updated(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, TxnInsertInsert row, + StdVec key, uint64 tuple, const TxnInsertRoom *room, const TxnInsertUniqueness *unique, + const TxnMemtableVersionShown *go, size_t *stopped) +{ + HostIoWaits waits = host_io_Waits_testing(); + TxnInsertReplaces replaces = {.key = key, .tuple = {tuple}, .lock = TXN_MEMTABLE_LOCK_MODE_NO_KEY_UPDATE}; + TxnInsertKeyed keyed = {.stripe = txn_change_stripe(&row.storage, row.key.ptr, row.key.len, true)}; + TxnInsertInserted inserted; + HostError error; + + replaces.stripe = txn_change_stripe(&row.storage, key.ptr, key.len, true); + if (unique != NULL) + { + keyed.has_unique = true; + keyed.unique = *unique; + } + if (!txn_insert_update(io, store, held, &row, &replaces, room, NULL, NULL, NULL, 0, keyed, &waits, + txn_insert_tests_updated_decide, (void *) go, NULL, stopped, &inserted, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2577, 95, &error); + return inserted; +} + +/* txn_insert_tests_replayed_rows replays a database's logged row changes of a relation on another storage, each by its kind: a put's by the insert's redo, an end's by the change's. */ +static void +txn_insert_tests_replayed_rows(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database, + uint32 relation) +{ + StdVec writes = txn_insert_tests_logged(database); + + for (size_t k = 0; k < writes.len; k++) + { + const txn_insert_tests_logged_t *write = &STD_VEC_AT(&writes, txn_insert_tests_logged_t, k); + const TxnMemtablePut *put; + HostError error; + + if (write->tag != TXN_INSERT_TESTS_LOGGED_ROW || write->u.row.storage.relation != relation) + continue; + if (txn_record_RowChange_put(&write->u.row, &put)) + { + if (!txn_insert_redo(io, store, held, &write->u.row, write->end, &TXN_INSERT_TESTS_PAGE, NULL, NULL, 0, + &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2589, 81, &error); + } + else if (!txn_change_redo(io, held, &write->u.row, write->end, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2590, 73, &error); + } + txn_insert_tests_logged_free(&writes); +} + +/* txn_insert_tests_usize_eq reports whether two usizes are equal, a Stopped's T compared. */ +static bool +txn_insert_tests_usize_eq(const void *a, const void *b) +{ + return *(const size_t *) a == *(const size_t *) b; +} + +/* txn_insert_tests_inserted_is asserts at a line and column that an update did what was wanted, a Stopped's count compared too. */ +static void +txn_insert_tests_inserted_is(uint32 line, uint32 col, const TxnInsertInserted *got, size_t got_stopped, + const TxnInsertInserted *want, size_t want_stopped) +{ + TxnInsertInsertedRef left = {got, {&got_stopped, std_usize_Debug_fmt}}; + TxnInsertInsertedRef right = {want, {&want_stopped, std_usize_Debug_fmt}}; + + STD_ASSERT_EQ(line, col, txn_insert_Inserted_PartialEq_eq(got, &got_stopped, want, &want_stopped, + txn_insert_tests_usize_eq), + &left, txn_insert_Inserted_Debug_fmt, &right, txn_insert_Inserted_Debug_fmt); +} + +/* txn_insert_tests_key returns a test's key by its number, its bytes big-endian. */ +static StdVec +txn_insert_tests_key(uint64 n) +{ + uint8 bytes[8]; + StdVec key; + + std_u64_to_be_bytes(n, bytes); + std_slice_to_vec_Checked(&key, bytes, sizeof(bytes), 1, 1); + return key; +} + +/* txn_insert_tests_key_ops returns what each version of a test's numbered key does. */ +static StdVec +txn_insert_tests_key_ops(HostIoIo *io, const HostSharedHeld *held, uint64 n) +{ + StdVec key = txn_insert_tests_key(n); + StdVec ops = txn_insert_tests_ops(io, host_shared_Held_record(held), key.ptr, key.len); + + std_vec_free_Checked(&key, 1, 1); + return ops; +} + +/* txn_insert_tests_update_made makes the update case's database's repository, a reader, and its storage's state. */ +static void +txn_insert_tests_update_made(HostIoIo *io, uint32 database, RepoRepo *repo, HostSharedReader *reading, + HostSharedHeld *held) +{ + txn_insert_tests_repository(io, database, 2606, 95, 2607, 50, repo, reading); + *held = txn_insert_tests_stated(io, repo, reading, txn_insert_tests_stored(database), 2609, 105); +} + +/* + * (4b)'s section 4, an update's forms: one that keeps its key pushes its new + * row's Put over the row, on the row's block while it has room, with no end + * of its own; one that moves its row to another key pushes the old key's Move + * with its own record before the new key's Put; a read that stops pushes and + * logs nothing and gives back what it stopped with; an immediate key's move + * onto a live row's key finds it; and the records replayed on another storage + * give each key's chain as the primary's. + */ +void +test_txn_insert_tests_an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary(void) +{ + static const TxnMemtableVersionShown command = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + RepoRepo replica_repo; + HostSharedReader replica_reading; + HostSharedHeld replica; + TreesStoreReader store; + TreesStoreReader replica_store; + TxnInsertRoom room = txn_insert_tests_room(28); + TxnInsertInsert row; + TxnInsertInserted inserted; + TxnInsertInserted want = {.tag = TXN_INSERT_INSERTED_TOOK}; + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + TxnInsertUniqueness unique = {.order = &order, .index = 16386}; + HostXid committed; + uint64 first; + uint64 other; + size_t stopped = 0; + size_t records; + StdVec writes; + StdVec changes; + + txn_insert_tests_update_made(io, 9340, &repo, &reading, &held); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + first = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9340), 1), 28, NULL, NULL, 0, + 2616, 81); + committed = simc_begin(); + simc_commit(committed); + row = txn_insert_tests_row(txn_insert_tests_stored(9340), 3); + row.writer.xid = committed.f0; + other = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, 28, NULL, NULL, 0, + 2620, 102); + inserted = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9340), 1), + txn_insert_tests_key(1), first, &room, NULL, &command, &stopped); + want.u.took.tuple = 3; + txn_insert_tests_inserted_is(2622, 9, &inserted, stopped, &want, 0); + { + txn_insert_tests_op_t ops[] = {{"put", 3}, {"put", first}}; + + txn_insert_tests_ops_are(2623, 9, txn_insert_tests_key_ops(io, &held, 1), ops, lengthof(ops)); + } + writes = txn_insert_tests_logged(9340); + records = writes.len; + txn_insert_tests_logged_free(&writes); + inserted = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9340), 2), + txn_insert_tests_key(1), 3, &room, NULL, NULL, &stopped); + want.tag = TXN_INSERT_INSERTED_STOPPED; + txn_insert_tests_inserted_is(2626, 9, &inserted, stopped, &want, 2); + writes = txn_insert_tests_logged(9340); + STD_ASSERT_EQ_U64(2627, 9, writes.len, records); + txn_insert_tests_logged_free(&writes); + inserted = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9340), 2), + txn_insert_tests_key(1), 3, &room, NULL, &command, &stopped); + want.tag = TXN_INSERT_INSERTED_TOOK; + want.u.took.tuple = 4; + txn_insert_tests_inserted_is(2629, 9, &inserted, stopped, &want, 0); + { + txn_insert_tests_op_t moved[] = {{"move", 0}, {"put", 3}, {"put", first}}; + txn_insert_tests_op_t put[] = {{"put", 4}}; + + txn_insert_tests_ops_are(2630, 9, txn_insert_tests_key_ops(io, &held, 1), moved, lengthof(moved)); + txn_insert_tests_ops_are(2631, 9, txn_insert_tests_key_ops(io, &held, 2), put, lengthof(put)); + } + writes = txn_insert_tests_logged(9340); + std_vec_new(&changes, sizeof(const TxnRecordRowChange *), _Alignof(const TxnRecordRowChange *)); + for (size_t k = records; k < writes.len; k++) + { + const txn_insert_tests_logged_t *write = &STD_VEC_AT(&writes, txn_insert_tests_logged_t, k); + const TxnRecordRowChange *change = &write->u.row; + + if (write->tag == TXN_INSERT_TESTS_LOGGED_ROW) + std_vec_push_Checked(&changes, &change, sizeof(change), _Alignof(const TxnRecordRowChange *)); + } + if (changes.len != 2) + { + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0), STD_FMT_LIT(" row changes")}; + StdFmtArg args[] = {{&changes.len, std_usize_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_panic_fmt(RUST_FILE, 2640, 71, &message); + } + { + const TxnRecordRowChange *ended = STD_VEC_AT(&changes, const TxnRecordRowChange *, 0); + const TxnRecordRowChange *put = STD_VEC_AT(&changes, const TxnRecordRowChange *, 1); + StdVec one = txn_insert_tests_key(1); + StdVec two = txn_insert_tests_key(2); + uint8 four[8]; + TxnMemtableOp end = {.tag = TXN_MEMTABLE_OP_MOVE, .u.move = {.tag = TXN_MEMTABLE_SUCCESSOR_KEY}}; + bool yes = true; + StdBytes ended_key = {ended->key.ptr, ended->key.len}; + StdBytes one_key = {one.ptr, one.len}; + StdBytes put_key = {put->key.ptr, put->key.len}; + StdBytes two_key = {two.ptr, two.len}; + + std_vec_clone(&end.u.move.u.key, &two, 1, 1); + std_u64_to_be_bytes(4, four); + std_vec_extend_from_slice_Checked(&end.u.move.u.key, four, sizeof(four), 1, 1); + { + StdTupleRef got = STD_TUPLE(3, STD_ITEM(&ended_key, std_bytes_Debug_fmt), + STD_ITEM(&ended->op, txn_memtable_Op_Debug_fmt), + STD_ITEM(&ended->no_key, std_bool_Debug_fmt)); + StdTupleRef wanted = STD_TUPLE(3, STD_ITEM(&one_key, std_bytes_Debug_fmt), + STD_ITEM(&end, txn_memtable_Op_Debug_fmt), + STD_ITEM(&yes, std_bool_Debug_fmt)); + + STD_ASSERT_EQ(2642, 9, trees_store_VecU8_PartialEq_eq(&ended->key, &one) && + txn_memtable_Op_PartialEq_eq(&ended->op, &end) && ended->no_key, &got, std_tuple_Debug_fmt, + &wanted, std_tuple_Debug_fmt); + } + STD_ASSERT_EQ(2643, 9, trees_store_VecU8_PartialEq_eq(&put->key, &two), &put_key, + std_bytes_Debug_fmt, &two_key, std_bytes_Debug_fmt); + txn_memtable_Op_drop(&end); + std_vec_free_Checked(&two, 1, 1); + std_vec_free_Checked(&one, 1, 1); + } + std_vec_free_Checked(&changes, sizeof(const TxnRecordRowChange *), _Alignof(const TxnRecordRowChange *)); + txn_insert_tests_logged_free(&writes); + inserted = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9340), 3), + txn_insert_tests_key(2), 4, &room, &unique, &command, &stopped); + STD_ASSERT(2647, 9, inserted.tag == TXN_INSERT_INSERTED_EXISTS, "matches!(onto, Inserted::Exists(..))"); + { + txn_insert_tests_op_t ops[] = {{"put", other}}; + + txn_insert_tests_ops_are(2648, 9, txn_insert_tests_key_ops(io, &held, 3), ops, lengthof(ops)); + } + txn_insert_tests_update_made(io, 9341, &replica_repo, &replica_reading, &replica); + host_shared_Reader_drop(&replica_reading); + replica_store = repo_Repo_reader(&replica_repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + txn_insert_tests_replayed_rows(io, TREES_STORE_READER_NODE_STORE(&replica_store), &replica, 9340, 16384); + for (uint64 n = 1; n <= 3; n++) + { + StdVec got = txn_insert_tests_key_ops(io, &replica, n); + StdVec primary = txn_insert_tests_key_ops(io, &held, n); + + txn_insert_tests_ops_is(2653, 13, &got, &primary); + } + txn_insert_tests_ended(io, 2655, 35); + host_shared_Held_drop(&replica); + repo_Repo_drop(&replica_repo); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_made_in returns the state of one of a database's relations' storages, unwrapped where the keyless case's made unwraps it. */ +static HostSharedHeld +txn_insert_tests_made_in(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, uint32 database, + uint32 relation) +{ + HostStorage storage = txn_insert_tests_stored(database); + + storage.relation = relation; + return txn_insert_tests_stated(io, repo, reading, storage, 2671, 93); +} + +/* txn_insert_tests_copy returns a keyless table's copy of a row by its number. */ +static TxnInsertInsert +txn_insert_tests_copy(uint64 n) +{ + TxnInsertInsert row = txn_insert_tests_row(txn_insert_tests_stored(9342), n); + + row.keyless = true; + return row; +} + +/* txn_insert_tests_rowed returns a deferrable key's row of relation 16390 by its number. */ +static TxnInsertInsert +txn_insert_tests_rowed(uint64 n) +{ + HostStorage storage = txn_insert_tests_stored(9342); + TxnInsertInsert row; + + storage.relation = 16390; + row = txn_insert_tests_row(storage, n); + row.deferred = true; + return row; +} + +/* txn_insert_tests_took_is asserts at a line and column that an update took a tuple id. */ +static void +txn_insert_tests_took_is(uint32 line, uint32 col, const TxnInsertInserted *got, size_t stopped, uint64 tid) +{ + TxnInsertInserted want = {.tag = TXN_INSERT_INSERTED_TOOK, .u.took.tuple = tid}; + + txn_insert_tests_inserted_is(line, col, got, stopped, &want, 0); +} + +/* + * (4b)'s sections 4 and 4a, the row-addressed forms: a keyless table's + * update removes the copy it replaces, naming the new copy's tuple id, then + * adds the new copy to the same key; a deferrable key's update that keeps its + * key replaces its row by its tuple id, and one that moves it deletes the row + * by its tuple id, naming the new row, and adds the new row to the new key; + * each replayed as the primary's. + */ +void +test_txn_insert_tests_a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids(void) +{ + static const TxnMemtableVersionShown command = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + static const uint32 relations[] = {16384, 16390}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + RepoRepo replica_repo; + HostSharedReader reading; + HostSharedHeld keyless; + HostSharedHeld deferred; + TreesStoreReader store; + TreesStoreReader replica_store; + TxnInsertRoom room = txn_insert_tests_room(28); + TxnInsertInserted took; + ChunksErrorError failed; + size_t stopped = 0; + uint64 first; + + txn_insert_tests_repository(io, 9342, 2666, 87, 2667, 46, &repo, &reading); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + keyless = txn_insert_tests_made_in(io, &repo, &reading, 9342, 16384); + deferred = txn_insert_tests_made_in(io, &repo, &reading, 9342, 16390); + first = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &keyless, txn_insert_tests_copy(1), 28, + NULL, NULL, 0, 2677, 71); + took = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &keyless, txn_insert_tests_copy(1), + txn_insert_tests_key(1), first, &room, NULL, &command, &stopped); + txn_insert_tests_took_is(2679, 9, &took, stopped, 2); + { + txn_insert_tests_op_t ops[] = {{"add", 2}, {"remove", first << 32 | 2}, {"add", first}}; + + txn_insert_tests_ops_are(2680, 9, txn_insert_tests_key_ops(io, &keyless, 1), ops, lengthof(ops)); + } + first = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &deferred, txn_insert_tests_rowed(1), 28, + NULL, NULL, 0, 2682, 73); + took = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &deferred, txn_insert_tests_rowed(1), + txn_insert_tests_key(1), first, &room, NULL, &command, &stopped); + txn_insert_tests_took_is(2684, 9, &took, stopped, 2); + { + txn_insert_tests_op_t ops[] = {{"update row", first << 32 | 2}, {"add row", first}}; + + txn_insert_tests_ops_are(2685, 9, txn_insert_tests_key_ops(io, &deferred, 1), ops, lengthof(ops)); + } + took = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &deferred, txn_insert_tests_rowed(2), + txn_insert_tests_key(1), 2, &room, NULL, &command, &stopped); + txn_insert_tests_took_is(2687, 9, &took, stopped, 3); + { + StdVec ops = txn_insert_tests_key_ops(io, &deferred, 1); + txn_insert_tests_op_t want = {"delete row", UINT64CONST(2) << 32 | 3}; + const txn_insert_tests_op_t *got = &STD_VEC_AT(&ops, txn_insert_tests_op_t, 0); + + if (ops.len == 0) + std_index_check(0, 0, RUST_FILE, 2688, 20); + STD_ASSERT_EQ(2688, 9, got->at == want.at && strcmp(got->name, want.name) == 0, got, + txn_insert_tests_op_Debug_fmt, &want, txn_insert_tests_op_Debug_fmt); + std_vec_free_Checked(&ops, sizeof(txn_insert_tests_op_t), _Alignof(txn_insert_tests_op_t)); + } + { + txn_insert_tests_op_t ops[] = {{"add row", 3}}; + + txn_insert_tests_ops_are(2689, 9, txn_insert_tests_key_ops(io, &deferred, 2), ops, lengthof(ops)); + } + if (!repo_Repo_create(io, 9343, HOST_PG_DEFAULT, &replica_repo, &failed)) + STD_RESULT_EXPECT_FAILED(2690, 95, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + replica_store = repo_Repo_reader(&replica_repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (size_t r = 0; r < lengthof(relations); r++) + { + const HostSharedHeld *primary = r == 0 ? &keyless : &deferred; + HostSharedHeld replica = txn_insert_tests_made_in(io, &replica_repo, &reading, 9343, relations[r]); + + txn_insert_tests_replayed_rows(io, TREES_STORE_READER_NODE_STORE(&replica_store), &replica, 9342, + relations[r]); + for (uint64 n = 1; n <= 2; n++) + { + StdVec got = txn_insert_tests_key_ops(io, &replica, n); + StdVec want = txn_insert_tests_key_ops(io, primary, n); + + txn_insert_tests_ops_is(2696, 17, &got, &want); + } + host_shared_Held_drop(&replica); + } + txn_insert_tests_ended(io, 2699, 35); + repo_Repo_drop(&replica_repo); + host_shared_Held_drop(&deferred); + host_shared_Held_drop(&keyless); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_shown_t is what a version shows: its raw command id and the command id that ended it. */ +typedef struct txn_insert_tests_shown_t +{ + uint32 raw; + uint32 ended; +} txn_insert_tests_shown_t; + +/* txn_insert_tests_shown_Debug_fmt is what a version shows's Debug, the tuple (raw, ended). */ +static bool +txn_insert_tests_shown_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_insert_tests_shown_t *shown = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&shown->raw, std_u32_Debug_fmt), + STD_ITEM(&shown->ended, std_u32_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_insert_tests_showns_Debug_fmt is a Vec<(u32, u32)>'s Debug. */ +static bool +txn_insert_tests_showns_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *shows = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < shows->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(shows, txn_insert_tests_shown_t, k), txn_insert_tests_shown_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_insert_tests_shows returns what each version of a test's numbered key shows, newest first. */ +static StdVec +txn_insert_tests_shows(HostIoIo *io, const HostSharedHeld *held, uint64 n) +{ + StdVec key = txn_insert_tests_key(n); + HostDeltaHold hold; + HostError error; + StdVec versions; + StdVec shows; + + if (!host_delta_hold(io, &hold, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2716, 46, &error); + if (!txn_memtable_table_versions(io, &hold, host_shared_Held_record(held), key.ptr, key.len, &versions, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2717, 79, &error); + std_vec_with_capacity_Checked(&shows, versions.len, sizeof(txn_insert_tests_shown_t), + _Alignof(txn_insert_tests_shown_t)); + for (size_t k = 0; k < versions.len; k++) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(&versions, + TxnMemtableVersionVersion, + k)); + txn_insert_tests_shown_t shown = {txn_memtable_version_Record_raw(&record), + txn_memtable_version_Record_ended(&record)}; + + std_vec_push_Checked(&shows, &shown, sizeof(shown), _Alignof(txn_insert_tests_shown_t)); + } + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + std_vec_free_Checked(&key, 1, 1); + return shows; +} + +/* txn_insert_tests_shows_are asserts at a line and column that what a key's versions show is what was wanted, and frees the list. */ +static void +txn_insert_tests_shows_are(uint32 line, uint32 col, StdVec shows, const txn_insert_tests_shown_t *want, + size_t want_len) +{ + StdVec wanted = {(void *) want, want_len, want_len}; + bool eq = shows.len == want_len && + memcmp(shows.ptr, want, want_len * sizeof(txn_insert_tests_shown_t)) == 0; + + STD_ASSERT_EQ(line, col, eq, &shows, txn_insert_tests_showns_Debug_fmt, &wanted, + txn_insert_tests_showns_Debug_fmt); + std_vec_free_Checked(&shows, sizeof(txn_insert_tests_shown_t), _Alignof(txn_insert_tests_shown_t)); +} + +/* + * (4b)'s section 6 and CommPF 7's ruling on a same-key update: an update of a + * row its own transaction wrote shows the combo command id its read gave on + * the row it ends, the new row's Put showing its own command for its own row + * and the combo id for the row it ended, and a move's end showing the combo + * id as its own. + */ +void +test_txn_insert_tests_an_updates_combo_command_shows_on_the_row_it_ends(void) +{ + static const txn_insert_tests_shown_t kept[] = {{1, 7}, {0, 0}}; + static const txn_insert_tests_shown_t both = {8, 8}; + static const txn_insert_tests_shown_t moved[] = {{2, 2}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + TxnInsertRoom room = txn_insert_tests_room(28); + TxnMemtableVersionShown combo = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMBO, .u.combo = 7}; + TxnInsertInsert row; + TxnInsertInserted took; + size_t stopped = 0; + uint64 first; + uint64 second; + + txn_insert_tests_repository(io, 9344, 2709, 87, 2710, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9344), 2711, 106); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + first = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, + txn_insert_tests_row(txn_insert_tests_stored(9344), 1), 28, NULL, NULL, 0, + 2720, 81); + row = txn_insert_tests_row(txn_insert_tests_stored(9344), 1); + row.writer.command = 1; + took = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, txn_insert_tests_key(1), + first, &room, NULL, &combo, &stopped); + if (took.tag != TXN_INSERT_INSERTED_TOOK) + { + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + TxnInsertInsertedRef shown = {&took, {&stopped, std_usize_Debug_fmt}}; + StdFmtArg args[] = {{&shown, txn_insert_Inserted_Debug_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_panic_fmt(RUST_FILE, 2723, 50, &message); + } + second = took.u.took.tuple; + txn_insert_tests_shows_are(2724, 9, txn_insert_tests_shows(io, &held, 1), kept, lengthof(kept)); + row = txn_insert_tests_row(txn_insert_tests_stored(9344), 2); + row.writer.command = 2; + combo.u.combo = 8; + took = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, txn_insert_tests_key(1), + second, &room, NULL, &combo, &stopped); + STD_ASSERT(2727, 9, took.tag == TXN_INSERT_INSERTED_TOOK, "matches!(took, Inserted::Took(_))"); + { + StdVec shows = txn_insert_tests_shows(io, &held, 1); + + const txn_insert_tests_shown_t *got; + + std_index_check(0, shows.len, RUST_FILE, 2728, 20); + got = &STD_VEC_AT(&shows, txn_insert_tests_shown_t, 0); + STD_ASSERT_EQ(2728, 9, got->raw == both.raw && got->ended == both.ended, got, + txn_insert_tests_shown_Debug_fmt, &both, txn_insert_tests_shown_Debug_fmt); + std_vec_free_Checked(&shows, sizeof(txn_insert_tests_shown_t), _Alignof(txn_insert_tests_shown_t)); + } + txn_insert_tests_shows_are(2729, 9, txn_insert_tests_shows(io, &held, 2), moved, lengthof(moved)); + txn_insert_tests_ended(io, 2730, 35); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_token_is asserts at a line and column that a key's newest version holds the token wanted, or none. */ +static void +txn_insert_tests_token_is(uint32 line, uint32 col, const StdVec *versions, uint32 index_line, uint32 index_col, + bool has_want, uint32 want) +{ + uint32 token = 0; + bool has; + StdOptionRef got; + StdOptionRef wanted = {has_want, {&want, std_u32_Debug_fmt}}; + + std_index_check(0, versions->len, RUST_FILE, index_line, index_col); + has = txn_memtable_version_Version_token(&STD_VEC_AT(versions, TxnMemtableVersionVersion, 0), &token); + got = (StdOptionRef) + { + has, + { + &token, std_u32_Debug_fmt + } + }; + STD_ASSERT_EQ(line, col, has == has_want && (!has || token == want), &got, std_option_Debug_fmt, &wanted, + std_option_Debug_fmt); +} + +/* txn_insert_tests_completion returns the completion of a speculative insert of a key at a tuple id, written by a writer. */ +static TxnRecordCompletion +txn_insert_tests_completion(TxnMemtableVersionWriter writer, uint64 key, uint64 tid) +{ + TxnRecordCompletion completion = {.writer = writer, .storage = txn_insert_tests_stored(9345), .tuple = {tid}}; + + completion.key = txn_insert_tests_key(key); + return completion; +} + +/* txn_insert_tests_completed runs a speculative insert's completion, logged or not, unwrapped at a line and column, and answers it. */ +static bool +txn_insert_tests_completed(HostIoIo *io, const HostSharedHeld *held, TxnRecordCompletion completion, + TxnInsertCompleted completed, bool logged, uint32 line, uint32 col) +{ + HostError error; + bool result; + + if (!txn_insert_complete(io, held, &completion, completed, logged, &result, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(line, col, &error); + std_vec_free_Checked(&completion.key, 1, 1); + return result; +} + +/* txn_insert_tests_bools_Debug_fmt is a Vec's Debug. */ +static bool +txn_insert_tests_bools_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *bools = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < bools->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(bools, bool, k), std_bool_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* + * (4b)'s speculative insert: an insert with a token pushes its version + * holding the token, which a Dirty read of the key reports with the running + * writer, and the insert's confirm clears it; a killed one is left out of + * what a checkpoint's runs write. + */ +void +test_txn_insert_tests_a_speculative_insert_holds_its_token_until_confirmed(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + HostXid running; + TxnMemtableVersionWriter writer; + TxnInsertInsert row; + uint64 first; + uint64 second; + HostDeltaHold hold; + StdVec versions; + SimcTaken dirty; + HostSnapshot lent; + TxnVisibleRule rule; + TxnVisibleSeen seen; + HostError error; + StdVec writes; + StdVec killed; + StdVec chains; + uint8 one[8]; + + txn_insert_tests_repository(io, 9345, 2740, 87, 2741, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9345), 2742, 106); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + running = simc_begin(); + writer = (TxnMemtableVersionWriter) + { + .xid = running.f0 + }; + row = txn_insert_tests_row(txn_insert_tests_stored(9345), 1); + row.writer = writer; + row.has_token = true; + row.token = 5; + first = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, 28, NULL, NULL, 0, + 2748, 110); + if (!host_delta_hold(io, &hold, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2749, 42, &error); + std_u64_to_be_bytes(1, one); + if (!txn_memtable_table_versions(io, &hold, host_shared_Held_record(&held), one, sizeof(one), &versions, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2750, 87, &error); + txn_insert_tests_token_is(2751, 9, &versions, 2751, 20, true, 5); + dirty = simc_snapshot(HOST_SNAPSHOT_KIND_DIRTY, false); + lent = simc_Taken_lent(&dirty); + rule.ops = &TXN_VISIBLE_SNAPSHOT_RULE_OPS; + rule.self = &lent; + if (!txn_visible_seen(io, rule, versions.ptr, versions.len, NULL, &seen, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2753, 70, &error); + std_index_check(0, seen.rows.len, RUST_FILE, 2754, 21); + { + const TxnVisibleDirty *found = &STD_VEC_AT(&seen.rows, TxnVisibleSeenRow, 0).dirty; + uint32 five = 5; + StdOptionRef xmin = {found->has_xmin, {&found->xmin, host_Xid_Debug_fmt}}; + StdOptionRef token = {found->has_token, {&found->token, std_u32_Debug_fmt}}; + StdOptionRef want_xmin = {true, {&running, host_Xid_Debug_fmt}}; + StdOptionRef want_token = {true, {&five, std_u32_Debug_fmt}}; + StdTupleRef got = STD_TUPLE(2, STD_ITEM(&xmin, std_option_Debug_fmt), STD_ITEM(&token, std_option_Debug_fmt)); + StdTupleRef want = STD_TUPLE(2, STD_ITEM(&want_xmin, std_option_Debug_fmt), + STD_ITEM(&want_token, std_option_Debug_fmt)); + + STD_ASSERT_EQ(2754, 9, found->has_xmin && found->xmin.f0 == running.f0 && found->has_token && + found->token == 5, &got, std_tuple_Debug_fmt, &want, std_tuple_Debug_fmt); + } + STD_ASSERT(2761, 9, txn_insert_tests_completed(io, &held, txn_insert_tests_completion(writer, 1, first), + TXN_INSERT_COMPLETED_CONFIRMED, true, 2761, 90), + "complete(io, &held, &completion(1, first), (Completed::Confirmed, true)).unwrap()"); + txn_insert_tests_token_is(2762, 9, &versions, 2762, 20, false, 0); + txn_visible_Seen_drop(&seen); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + row = txn_insert_tests_row(txn_insert_tests_stored(9345), 2); + row.writer = writer; + row.has_token = true; + row.token = 6; + second = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, 28, NULL, NULL, 0, + 2766, 110); + STD_ASSERT(2767, 9, txn_insert_tests_completed(io, &held, txn_insert_tests_completion(writer, 2, second), + TXN_INSERT_COMPLETED_KILLED, true, 2767, 88), + "complete(io, &held, &completion(2, second), (Completed::Killed, true)).unwrap()"); + STD_ASSERT(2768, 9, !txn_insert_tests_completed(io, &held, txn_insert_tests_completion(writer, 3, second), + TXN_INSERT_COMPLETED_KILLED, false, 2768, 90), + "!complete(io, &held, &completion(3, second), (Completed::Killed, false)).unwrap()"); + writes = txn_insert_tests_logged(9345); + std_vec_new(&killed, sizeof(bool), _Alignof(bool)); + for (size_t k = 0; k < writes.len; k++) + { + const txn_insert_tests_logged_t *write = &STD_VEC_AT(&writes, txn_insert_tests_logged_t, k); + + if (write->tag == TXN_INSERT_TESTS_LOGGED_ROW) + std_vec_push_Checked(&killed, &write->u.row.speculative, sizeof(bool), _Alignof(bool)); + } + { + static const bool both[] = {true, true}; + StdVec wanted = {(void *) both, 2, 2}; + bool eq = killed.len == 2 && STD_VEC_AT(&killed, bool, 0) && STD_VEC_AT(&killed, bool, 1); + + STD_ASSERT_EQ(2773, 9, eq, &killed, txn_insert_tests_bools_Debug_fmt, &wanted, + txn_insert_tests_bools_Debug_fmt); + } + std_vec_free_Checked(&killed, sizeof(bool), _Alignof(bool)); + txn_insert_tests_logged_free(&writes); + chains = txn_insert_tests_chains(io, host_shared_Held_record(&held)); + { + txn_insert_tests_chain_t want = {.key = {one, sizeof(one), sizeof(one)}, .versions = 1}; + StdVec wanted = {&want, 1, 1}; + + STD_ASSERT_EQ(2774, 9, txn_insert_tests_chains_eq(&chains, &wanted), &chains, txn_insert_tests_chains_Debug_fmt, &wanted, + txn_insert_tests_chains_Debug_fmt); + } + txn_insert_tests_chains_free(&chains); + txn_insert_tests_ended(io, 2775, 35); + simc_Taken_drop(&dirty); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* + * An aborted update that moves its row goes whole at its take-back: the new + * key's put and the old key's move, since turn records both keys in its list + * before its prepare, and the committed row's put stays its key's one + * version. + */ +void +test_txn_insert_tests_an_aborted_updates_new_row_and_end_go_at_the_take_back(void) +{ + static const TxnMemtableVersionShown command = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + TxnInsertRoom room = txn_insert_tests_room(28); + TxnInsertInsert row; + TxnInsertInserted moved; + HostXid committed; + HostXid aborted; + HostError error; + size_t stopped = 0; + uint64 first; + + txn_insert_tests_repository(io, 9389, 2785, 87, 2786, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, txn_insert_tests_stored(9389), 2787, 106); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + committed = simc_begin(); + aborted = simc_begin(); + row = txn_insert_tests_row(txn_insert_tests_stored(9389), 1); + row.writer.xid = committed.f0; + first = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, 28, NULL, NULL, 0, + 2792, 66); + simc_commit(committed); + txn_insert_tests_ended(io, 2794, 28); + row = txn_insert_tests_row(txn_insert_tests_stored(9389), 2); + row.writer.xid = aborted.f0; + moved = txn_insert_tests_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, txn_insert_tests_key(1), + first, &room, NULL, &command, &stopped); + STD_ASSERT(2798, 9, moved.tag == TXN_INSERT_INSERTED_TOOK, "matches!(moved, Inserted::Took(_))"); + { + StdVec ops = txn_insert_tests_key_ops(io, &held, 1); + StdStr got; + StdStr want = {"move", 4}; + + std_index_check(0, ops.len, RUST_FILE, 2799, 20); + got = (StdStr) + { + STD_VEC_AT(&ops, txn_insert_tests_op_t, 0).name, + strlen(STD_VEC_AT(&ops, txn_insert_tests_op_t, 0).name) + }; + STD_ASSERT_EQ(2799, 9, strcmp(got.ptr, "move") == 0, &got, std_str_Debug_fmt, &want, + std_str_Debug_fmt); + std_vec_free_Checked(&ops, sizeof(txn_insert_tests_op_t), _Alignof(txn_insert_tests_op_t)); + } + simc_abort(aborted); + if (!txn_written_aborted(io, 0, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2801, 30, &error); + { + txn_insert_tests_op_t put[] = {{"put", first}}; + + txn_insert_tests_ops_are(2802, 9, txn_insert_tests_key_ops(io, &held, 1), put, lengthof(put)); + txn_insert_tests_ops_are(2803, 9, txn_insert_tests_key_ops(io, &held, 2), NULL, 0); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_Folded_Equal_equal is Folded's Equal: keys are equal when their bytes are. */ +static bool +txn_insert_tests_Folded_Equal_equal(void *self, HostIoIo *io, const uint8 *a, size_t a_len, const uint8 *b, size_t b_len, + bool *result, HostError *error) +{ + *result = a_len == b_len && memcmp(a, b, a_len) == 0; + return true; +} + +/* + * TXN_INSERT_TESTS_FOLDED_OPS is Folded's Equal, a test's class equality + * that calls keys equal when their bytes are, so a resolve takes the path of + * a key whose class compares by a function, its stripe the storage's, with no + * key equal to another. + */ +static const TxnInsertEqualOps TXN_INSERT_TESTS_FOLDED_OPS = {txn_insert_tests_Folded_Equal_equal}; + +/* txn_insert_tests_resolver is the stripe case's resolver: an immediate key's insert under the storage's stripe takes (0,4). */ +static bool +txn_insert_tests_resolver(void *captures, HostIoIo *io, HostError *error) +{ + static const StdFmtPiece pieces[] = {STD_FMT_LIT("the resolve gave "), STD_FMT_ARG(0)}; + HostStorage storage = txn_insert_tests_stored(9390); + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + TxnInsertRoom room = txn_insert_tests_room(28); + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TxnInsertInserted inserted; + bool ok; + + if (!txn_insert_tests_opened(io, storage, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + TxnInsertKeyed keyed = {.stripe = txn_change_stripe(&storage, NULL, 0, false), .has_unique = true, + .unique = {.order = &order, .index = 16386, .has_equal = true, .equal = {&TXN_INSERT_TESTS_FOLDED_OPS, NULL}}}; + HostIoWaits waits = host_io_Waits_testing(); + TxnInsertInsert row = txn_insert_tests_row(storage, 100); + + ok = txn_insert_insert(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &room, NULL, NULL, NULL, 0, + keyed, &waits, &inserted, error); + } + if (ok && !(inserted.tag == TXN_INSERT_INSERTED_TOOK && inserted.u.took.tuple == 4)) + { + TxnInsertInsertedRef other = {&inserted, {NULL, std_unit_Debug_fmt}}; + StdFmtArg args[] = {{&other, txn_insert_Inserted_Debug_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + *error = txn_insert_tests_gathering_fork(&arguments); + ok = false; + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* txn_insert_tests_inserteds_Debug_fmt is a Vec>'s Debug. */ +static bool +txn_insert_tests_inserteds_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *inserteds = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < inserteds->len; k++) + { + TxnInsertInsertedRef one = {&STD_VEC_AT(inserteds, TxnInsertInserted, k), {NULL, std_unit_Debug_fmt}}; + + std_DebugList_entry(&list, &one, txn_insert_Inserted_Debug_fmt); + } + return std_DebugList_finish(&list); +} + +/* txn_insert_tests_pusher is the stripe case's pusher: four keyless-striped inserts, the last after the resolve's. */ +static bool +txn_insert_tests_pusher(void *captures, HostIoIo *io, HostError *error) +{ + static const StdFmtPiece pieces[] = {STD_FMT_LIT("the pushes gave "), STD_FMT_ARG(0)}; + static const uint64 tids[] = {1, 2, 3, 5}; + HostStorage storage = txn_insert_tests_stored(9390); + TxnInsertRoom room = txn_insert_tests_room(28); + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + StdVec took; + bool ok = true; + + if (!txn_insert_tests_opened(io, storage, &repo, &reading, &held, error)) + return false; + std_vec_new(&took, sizeof(TxnInsertInserted), _Alignof(TxnInsertInserted)); + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + HostIoWaits waits = host_io_Waits_testing(); + + for (uint64 n = 1; ok && n <= 4; n++) + { + TxnInsertKeyed keyed = {.stripe = txn_change_stripe(&storage, NULL, 0, false)}; + TxnInsertInsert row = txn_insert_tests_row(storage, n); + TxnInsertInserted inserted; + + ok = txn_insert_insert(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &room, NULL, NULL, NULL, 0, + keyed, &waits, &inserted, error); + if (ok) + std_vec_push_Checked(&took, &inserted, sizeof(inserted), _Alignof(TxnInsertInserted)); + } + } + if (ok) + { + bool eq = took.len == lengthof(tids); + + for (size_t k = 0; eq && k < took.len; k++) + eq = STD_VEC_AT(&took, TxnInsertInserted, k).tag == TXN_INSERT_INSERTED_TOOK && + STD_VEC_AT(&took, TxnInsertInserted, k).u.took.tuple == tids[k]; + if (!eq) + { + StdFmtArg args[] = {{&took, txn_insert_tests_inserteds_Debug_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + *error = txn_insert_tests_gathering_fork(&arguments); + ok = false; + } + } + std_vec_free_Checked(&took, sizeof(TxnInsertInserted), _Alignof(TxnInsertInserted)); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * (4b)'s section 3's hold-back flag: a resolve over its storage's stripe, + * held at its read while another writer of the storage pushes a row three + * times, starts over three times and holds the stripe back at the third, so + * that writer's next insert sleeps before its read until the resolve + * finishes, within 3 plus 2 rounds, and takes the tuple id after the + * resolve's. + */ +void +test_txn_insert_tests_a_resolve_whose_stripe_moves_three_times_holds_its_writers_back(void) +{ + static const SimcStep steps[] = {{"pusher", 6, UINT64_MAX}, {"resolver", 8, UINT64_MAX}, {"pusher", 6, UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + SimcActor resolver = {txn_insert_tests_resolver}; + SimcActor pusher = {txn_insert_tests_pusher}; + RepoRepo repo; + ChunksErrorError failed; + SimcRun run; + SimcReport report; + + if (!repo_Repo_create(io, 9390, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(2825, 81, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + repo_Repo_drop(&repo); + run = simc_Run_new(1); + simc_Run_actor(&run, "resolver", 8, resolver); + simc_Run_actor(&run, "pusher", 6, pusher); + for (int k = 0; k < 3; k++) + { + simc_Run_until(&run, "resolver", 8, "doltgres-insert-resolved"); + simc_Run_until(&run, "pusher", 6, "doltgres-memtable-unpinned"); + } + simc_Run_until(&run, "resolver", 8, "doltgres-insert-resolved"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, RUST_FILE, 2863, 67, 2864, 9); + simc_Report_drop(&report); + simc_TestIo_drop(&test_io); +} + +/* txn_insert_tests_go is an update's read that goes on as the transaction's command, naming no row. */ +static bool +txn_insert_tests_go(void *captures, HostIoIo *io, TxnMemtableVersionVersion *versions, size_t versions_len, + TxnChangeRead *result, void *stopped, HostError *error) +{ + *result = (TxnChangeRead) + { + .tag = TXN_CHANGE_READ_GO, .u.go.shown.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND + }; + return true; +} + +/* txn_insert_tests_kept_t is kept_across's updater's captures: its database and its row's tuple id. */ +typedef struct txn_insert_tests_kept_t +{ + uint32 database; + uint64 tid; +} txn_insert_tests_kept_t; + +/* txn_insert_tests_kept_updater is kept_across's updater: an update of the row that keeps its two out-of-line values. */ +static bool +txn_insert_tests_kept_updater(void *captures, HostIoIo *io, HostError *error) +{ + static const size_t kept[] = {0, 1}; + const txn_insert_tests_kept_t *c = captures; + HostStorage storage = txn_insert_tests_stored(c->database); + HostStorage toasted = storage; + TxnInsertRoom room = txn_insert_tests_room(60); + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld toast; + bool ok; + + toasted.relation = 16390; + if (!txn_insert_tests_opened(io, storage, &repo, &reading, &held, error)) + return false; + if (!txn_state_state(io, &repo, &reading, &toasted, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &toast, error)) + { + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return false; + } + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + TxnValuesEntry entries[2]; + TxnInsertInsert row = txn_insert_tests_valued(c->database, entries); + TxnInsertReplaces replaces = {.tuple = {c->tid}, .lock = TXN_MEMTABLE_LOCK_MODE_NO_KEY_UPDATE}; + TxnInsertKeyed keyed = {.stripe = txn_change_stripe(&storage, row.key.ptr, row.key.len, true)}; + HostIoWaits waits = host_io_Waits_testing(); + TxnInsertInserted inserted; + + std_vec_free_Checked(&row.kept, sizeof(size_t), _Alignof(size_t)); + std_slice_to_vec_Checked(&row.kept, kept, lengthof(kept), sizeof(size_t), _Alignof(size_t)); + std_vec_clone(&replaces.key, &row.key, 1, 1); + replaces.stripe = txn_change_stripe(&storage, replaces.key.ptr, replaces.key.len, true); + ok = txn_insert_update(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &replaces, &room, NULL, &toast, + NULL, 0, keyed, &waits, txn_insert_tests_go, NULL, NULL, NULL, &inserted, error); + } + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * txn_insert_tests_kept_across runs an update that keeps a row's two + * out-of-line values while a checkpoint runs, placed by a script: with the + * update held between its count adds' push and its row's push, the + * checkpoint's runs hold the adds and not the row; run before the update, + * neither; after it, both. A crash and redo of every logged record then leave + * each kept value's count at 2, and a second crash during that redo and redo + * again leave it at 2. + */ +static void +txn_insert_tests_kept_across(uint32 database, void (*script) (SimcRun *run)) +{ + static const uint64 twice[] = {2, 2}; + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostStorage storage = txn_insert_tests_stored(database); + HostStorage toasted = storage; + txn_insert_tests_kept_t c = {.database = database}; + SimcActor checkpointer = {txn_insert_tests_checkpointer}; + SimcActor updater = {txn_insert_tests_kept_updater, &c}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostSharedHeld toast; + HostSharedHeld deltas; + TreesStoreReader store; + TxnValuesEntry entries[2]; + TxnInsertInsert row; + SimcRun run; + SimcReport report; + HostError error; + + toasted.relation = 16390; + txn_insert_tests_repository(io, database, 2878, 59, 2879, 46, &repo, &reading); + row = txn_insert_tests_valued(database, entries); + deltas = txn_insert_tests_written(io, database, row.writer, entries); + host_shared_Held_drop(&deltas); + held = txn_insert_tests_stated(io, &repo, &reading, storage, 2884, 99); + toast = txn_insert_tests_toasted(io, &repo, &reading, database, 2885, 100); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + c.tid = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, row, 60, &toast, entries, 2, + 2887, 78); + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + simc_checkpointed(UINT64CONST(0x100000000)); + run = simc_Run_new(1); + simc_Run_actor(&run, "checkpointer", 12, checkpointer); + simc_Run_actor(&run, "updater", 7, updater); + script(&run); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, RUST_FILE, 2915, 67, 2916, 9); + simc_Report_drop(&report); + if (!txn_memtable_branch_deltas(io, database, HOST_PG_DEFAULT, &deltas, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2918, 65, &error); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (int k = 0; k < 2; k++) + { + StdVec writes; + uint64 counts[2]; + + if (!txn_state_unlinked(io, &storage, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2921, 37, &error); + if (!txn_state_unlinked(io, &toasted, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2922, 37, &error); + if (!txn_runs_loaded(io, &TXN_INSERT_TESTS_PAGE, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(2923, 71, &error); + held = txn_insert_tests_stated(io, &repo, &reading, storage, 2924, 99); + toast = txn_insert_tests_toasted(io, &repo, &reading, database, 2925, 100); + writes = txn_insert_tests_logged(database); + txn_insert_tests_redone(io, TREES_STORE_READER_NODE_STORE(&store), &held, &toast, &deltas, &writes); + txn_insert_tests_counts(io, TREES_STORE_READER_NODE_STORE(&store), &toast, 2927, 95, + 2927, 104, counts); + txn_insert_tests_u64s_is(2928, 13, counts, 2, twice, 2); + host_shared_Held_drop(&toast); + host_shared_Held_drop(&held); + } + txn_insert_tests_ended(io, 2930, 35); + host_shared_Held_drop(&deltas); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* txn_insert_tests_kept_between holds the update between its count adds' push and its row's push while the checkpoint runs. */ +static void +txn_insert_tests_kept_between(SimcRun *run) +{ + static const SimcStep steps[] = {{"checkpointer", 12, UINT64_MAX}, {"updater", 7, UINT64_MAX}, + {"checkpointer", 12, UINT64_MAX}}; + + simc_Run_until(run, "updater", 7, "doltgres-insert-logged"); + simc_Run_script(run, steps, lengthof(steps)); +} + +/* + * (4b)'s section 5: an update held between its kept values' count adds and + * its row's push while a checkpoint sweeps its storage's rows and then its + * maps, a crash, and redo leave each kept value's count at 2, not 3, and a + * second crash during that redo too. + */ +void +test_txn_insert_tests_a_checkpoint_between_an_updates_count_adds_and_its_row_leaves_the_counts_once(void) +{ + txn_insert_tests_kept_across(9391, txn_insert_tests_kept_between); +} + +/* txn_insert_tests_kept_before runs the checkpoint before the update. */ +static void +txn_insert_tests_kept_before(SimcRun *run) +{ + static const SimcStep steps[] = {{"checkpointer", 12, UINT64_MAX}, {"updater", 7, UINT64_MAX}}; + + simc_Run_script(run, steps, lengthof(steps)); +} + +/* txn_insert_tests_kept_after runs the checkpoint after the update. */ +static void +txn_insert_tests_kept_after(SimcRun *run) +{ + static const SimcStep steps[] = {{"updater", 7, UINT64_MAX}, {"checkpointer", 12, UINT64_MAX}}; + + simc_Run_script(run, steps, lengthof(steps)); +} + +/* + * (4b)'s section 5: a checkpoint before an update that keeps two values, or + * after it, a crash, and redo leave each kept value's count at 2. + */ +void +test_txn_insert_tests_a_checkpoint_before_or_after_an_update_leaves_its_kept_counts_once(void) +{ + txn_insert_tests_kept_across(9392, txn_insert_tests_kept_before); + txn_insert_tests_kept_across(9393, txn_insert_tests_kept_after); +} + +/* txn_insert_tests_moving_updater is the key change case's updater: an update that moves its row from key 1 to key 2. */ +static bool +txn_insert_tests_moving_updater(void *captures, HostIoIo *io, HostError *error) +{ + const uint64 *tid = captures; + HostStorage storage = txn_insert_tests_stored(9394); + TxnInsertRoom room = txn_insert_tests_room(28); + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + bool ok; + + if (!txn_insert_tests_opened(io, storage, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + TxnInsertInsert row = txn_insert_tests_row(storage, 2); + TxnInsertReplaces replaces = {.key = txn_insert_tests_key(1), .tuple = {*tid}, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE}; + TxnInsertKeyed keyed = {.stripe = txn_change_stripe(&storage, row.key.ptr, row.key.len, true)}; + HostIoWaits waits = host_io_Waits_testing(); + TxnInsertInserted inserted; + + replaces.stripe = txn_change_stripe(&storage, replaces.key.ptr, replaces.key.len, true); + ok = txn_insert_update(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &replaces, &room, NULL, NULL, + NULL, 0, keyed, &waits, txn_insert_tests_go, NULL, NULL, NULL, &inserted, error); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * (4b)'s section 10: an update that moves its row from key 1 to key 2, held + * between its old key's move and its new key's put while a checkpoint runs, + * so the runs hold the move and not the put, a crash, and redo of every + * logged record leave both chains as the primary's, and a second crash + * during that redo too. + */ +void +test_txn_insert_tests_a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both(void) +{ + static const SimcStep steps[] = {{"checkpointer", 12, UINT64_MAX}, {"updater", 7, UINT64_MAX}, + {"checkpointer", 12, UINT64_MAX}}; + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostStorage storage = txn_insert_tests_stored(9394); + uint64 tid; + SimcActor checkpointer = {txn_insert_tests_checkpointer}; + SimcActor updater = {txn_insert_tests_moving_updater, &tid}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TreesStoreReader store; + StdVec wanted; + StdVec primary; + SimcRun run; + SimcReport report; + HostError error; + + txn_insert_tests_repository(io, 9394, 2962, 59, 2963, 46, &repo, &reading); + held = txn_insert_tests_stated(io, &repo, &reading, storage, 2966, 99); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + tid = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &held, txn_insert_tests_row(storage, 1), 28, + NULL, NULL, 0, 2968, 64); + host_shared_Held_drop(&held); + simc_checkpointed(UINT64CONST(0x100000000)); + run = simc_Run_new(1); + simc_Run_actor(&run, "checkpointer", 12, checkpointer); + simc_Run_actor(&run, "updater", 7, updater); + simc_Run_until(&run, "updater", 7, "doltgres-insert-logged"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, RUST_FILE, 2993, 67, 2994, 9); + simc_Report_drop(&report); + held = txn_insert_tests_stated(io, &repo, &reading, storage, 2997, 99); + primary = txn_insert_tests_chains(io, host_shared_Held_record(&held)); + host_shared_Held_drop(&held); + std_vec_new(&wanted, sizeof(txn_insert_tests_chain_t), _Alignof(txn_insert_tests_chain_t)); + for (uint64 n = 1; n <= 2; n++) + { + txn_insert_tests_chain_t chain = {.key = txn_insert_tests_key(n), .versions = 3 - n}; + + std_vec_push_Checked(&wanted, &chain, sizeof(chain), _Alignof(txn_insert_tests_chain_t)); + } + STD_ASSERT_EQ(3000, 9, txn_insert_tests_chains_eq(&primary, &wanted), &primary, + txn_insert_tests_chains_Debug_fmt, &wanted, txn_insert_tests_chains_Debug_fmt); + txn_insert_tests_chains_free(&wanted); + store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + for (int k = 0; k < 2; k++) + { + StdVec chains; + + if (!txn_state_unlinked(io, &storage, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(3003, 37, &error); + if (!txn_runs_loaded(io, &TXN_INSERT_TESTS_PAGE, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(3004, 71, &error); + held = txn_insert_tests_stated(io, &repo, &reading, storage, 3005, 99); + txn_insert_tests_replayed_rows(io, TREES_STORE_READER_NODE_STORE(&store), &held, 9394, 16384); + chains = txn_insert_tests_chains(io, host_shared_Held_record(&held)); + STD_ASSERT_EQ(3007, 13, txn_insert_tests_chains_eq(&chains, &primary), &chains, + txn_insert_tests_chains_Debug_fmt, &primary, txn_insert_tests_chains_Debug_fmt); + txn_insert_tests_chains_free(&chains); + host_shared_Held_drop(&held); + } + txn_insert_tests_ended(io, 3009, 35); + txn_insert_tests_chains_free(&primary); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* txn_insert_tests_resolving_t is resolving's resolver's captures: its storage's relation and the block it tries first, when given. */ +typedef struct txn_insert_tests_resolving_t +{ + uint32 relation; + bool has_target; + uint32 target; +} txn_insert_tests_resolving_t; + +/* txn_insert_tests_resolving_resolver is resolving's resolver: an insert of row 1 resolved over its storage's stripe, which takes a tuple id. */ +static bool +txn_insert_tests_resolving_resolver(void *captures, HostIoIo *io, HostError *error) +{ + static const StdFmtPiece pieces[] = {STD_FMT_LIT("the resolve gave "), STD_FMT_ARG(0)}; + const txn_insert_tests_resolving_t *c = captures; + HostStorage storage = txn_insert_tests_stored(9397); + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + TxnInsertRoom room = txn_insert_tests_room(28); + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + TxnInsertInserted inserted; + bool ok; + + storage.relation = c->relation; + if (!txn_insert_tests_opened(io, storage, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + TxnInsertKeyed keyed = {.stripe = txn_change_stripe(&storage, NULL, 0, false), .has_unique = true, + .unique = {.order = &order, .index = 16386, .has_equal = true, .equal = {&TXN_INSERT_TESTS_FOLDED_OPS, NULL}}}; + HostIoWaits waits = host_io_Waits_testing(); + TxnInsertInsert row = txn_insert_tests_row(storage, 1); + + row.has_target = c->has_target; + row.target = c->target; + ok = txn_insert_insert(io, TREES_STORE_READER_NODE_STORE(&store), &held, &row, &room, NULL, NULL, NULL, 0, + keyed, &waits, &inserted, error); + } + if (ok && inserted.tag != TXN_INSERT_INSERTED_TOOK) + { + TxnInsertInsertedRef other = {&inserted, {NULL, std_unit_Debug_fmt}}; + StdFmtArg args[] = {{&other, txn_insert_Inserted_Debug_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + *error = txn_insert_tests_gathering_fork(&arguments); + ok = false; + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * txn_insert_tests_resolving runs one insert of row 1 into a storage of + * database 9397 by its relation, resolved over the storage's stripe, trying + * the block target first when given, and terminates it past yields when + * given. + */ +static SimcReport +txn_insert_tests_resolving(uint32 relation, bool has_target, uint32 target, bool has_yields, uint64 yields) +{ + txn_insert_tests_resolving_t c = {.relation = relation, .has_target = has_target, .target = target}; + SimcActor resolver = {txn_insert_tests_resolving_resolver, &c}; + SimcRun run = simc_Run_new(1); + + simc_Run_actor(&run, "resolver", 8, resolver); + if (has_yields) + simc_Run_terminate_at(&run, "resolver", 8, yields); + return simc_Run_finish(run); +} + +/* txn_insert_tests_returned_is asserts at a line and column that a run ended with every actor returning Ok, its message the run's outcome. */ +static void +txn_insert_tests_returned_is(uint32 line, uint32 col, const SimcReport *report) +{ + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&report->outcome, simc_Outcome_Debug_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + if (!txn_vacuum_tests_all_returned(report)) + std_panic_fmt(RUST_FILE, line, col, &message); +} + +/* + * Bugs row 86: a resolve over its storage's stripe whose first target is + * refused goes on to its next target in the same pass, since its refused turn + * pushed nothing and so left the stripe's push count as the resolve read it, + * and the insert ends within twice the yields a plain insert of its row takes. + */ +void +test_txn_insert_tests_a_resolve_whose_first_target_is_refused_ends_in_bounded_turns(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + ChunksErrorError failed; + SimcReport plain; + SimcReport refused; + StdVec trace; + uint64 bound; + + if (!repo_Repo_create(io, 9397, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(3044, 81, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + repo_Repo_drop(&repo); + plain = txn_insert_tests_resolving(16384, false, 0, false, 0); + txn_insert_tests_returned_is(3050, 9, &plain); + trace = simc_Report_trace(&plain); + bound = 2 * (uint64) trace.len; + std_vec_free_Checked(&trace, sizeof(size_t), _Alignof(size_t)); + simc_Report_drop(&plain); + refused = txn_insert_tests_resolving(16388, true, 5, true, bound); + txn_insert_tests_returned_is(3054, 9, &refused); + simc_Report_drop(&refused); + simc_TestIo_drop(&test_io); +} + +/* + * Bugs row 97: an insert written frozen, as COPY FREEZE writes it, takes no + * room in the commit status table, its writer FrozenTransactionId having no + * entry, and its row change replays on another storage the same way: the + * replay pushes the row and asks the table for nothing. + */ +void +test_txn_insert_tests_a_frozen_inserts_row_change_replays_with_no_status_room(void) +{ + static const uint32 databases[] = {9464, 9465}; + static const uint8 one[8] = {0, 0, 0, 0, 0, 0, 0, 1}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedReader reading; + RepoRepo repos[2]; + HostSharedHeld helds[2]; + TxnInsertInsert frozen = txn_insert_tests_row(txn_insert_tests_stored(9464), 1); + TreesStoreReader store; + TreesStoreReader replica; + HostDeltaHold hold; + StdVec versions; + StdVec tupled; + HostRow tuple; + HostError error; + bool some; + + if (!host_shared_read(io, &reading, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(3064, 46, &error); + for (size_t k = 0; k < lengthof(databases); k++) + { + ChunksErrorError failed; + + if (!repo_Repo_create(io, databases[k], HOST_PG_DEFAULT, &repos[k], &failed)) + STD_RESULT_EXPECT_FAILED(3067, 95, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + helds[k] = txn_insert_tests_stated(io, &repos[k], &reading, txn_insert_tests_stored(databases[k]), + 3069, 107); + } + frozen.writer = TXN_MEMTABLE_VERSION_NO_TRANSACTION; + frozen.frozen = true; + store = repo_Repo_reader(&repos[0], HOST_CACHE_PLACEMENT_SERVER, &reading); + tuple.f0 = txn_insert_tests_took(io, TREES_STORE_READER_NODE_STORE(&store), &helds[0], frozen, 28, NULL, NULL, 0, + 3075, 72); + replica = repo_Repo_reader(&repos[1], HOST_CACHE_PLACEMENT_SERVER, &reading); + txn_insert_tests_replayed_rows(io, TREES_STORE_READER_NODE_STORE(&replica), &helds[1], 9464, 16384); + if (!host_delta_hold(io, &hold, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(3078, 42, &error); + if (!txn_memtable_table_versions(io, &hold, host_shared_Held_record(&helds[1]), one, sizeof(one), &versions, + &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(3079, 94, &error); + { + TxnMemtableVersionWriter none = TXN_MEMTABLE_VERSION_NO_TRANSACTION; + bool matched = versions.len == 1; + + if (matched) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(&versions, + TxnMemtableVersionVersion, + 0)); + TxnMemtableVersionWriter writer = txn_memtable_version_Record_writer(&record); + + matched = txn_memtable_version_Writer_PartialEq_eq(&writer, &none); + } + STD_ASSERT(3080, 9, matched, + "matches!(versions[..], [ref version] if version.record().writer() == crate::memtable::version::NO_TRANSACTION)"); + } + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + if (!txn_memtable_tuples_get(io, host_shared_Held_record(&helds[1]), tuple, &some, &tupled, &error)) + TXN_INSERT_TESTS_UNWRAP_FAILED(3084, 68, &error); + txn_insert_tests_row_is(3084, 9, some, &tupled); + if (some) + std_vec_free_Checked(&tupled, 1, 1); + txn_insert_tests_ended(io, 3085, 35); + for (size_t k = lengthof(databases); k-- > 0;) + { + host_shared_Held_drop(&helds[k]); + repo_Repo_drop(&repos[k]); + } + host_shared_Reader_drop(&reading); + simc_TestIo_drop(&test_io); +} +#endif diff --git a/src/doltgres/txn/c/memtable/branch.c b/src/doltgres/txn/c/memtable/branch.c index f39dfcf6c3e3f..bb9d233b7e94b 100644 --- a/src/doltgres/txn/c/memtable/branch.c +++ b/src/doltgres/txn/c/memtable/branch.c @@ -50,9 +50,8 @@ /* TXN_MEMTABLE_BRANCH_HEAD is HEAD, the length of a built node's first piece's head: its writer's full xid and its value's length. */ #define TXN_MEMTABLE_BRANCH_HEAD 12 -/* The delta's atomics only this file reads: its repository, its identity memtable's word, and its own tablespace. */ +/* The delta's atomics only this file reads: its repository and its own tablespace. */ #define TXN_MEMTABLE_BRANCH_REPOSITORY 0 -#define TXN_MEMTABLE_BRANCH_IDENTITY_WORD 2 #define TXN_MEMTABLE_BRANCH_OWN 6 /* txn_memtable_branch_internal returns an internal error of a working state's delta. */ @@ -554,14 +553,16 @@ txn_memtable_branch_reclaim_pad_0(void *frame_) } } -/* txn_memtable_branch_named reports whether a delta's words name a memtable or its chunk spills' manifest lists or pends a file. */ +/* txn_memtable_branch_named reports whether a delta's words name a memtable or its chunk or identity spills' manifest lists or pends a file. */ static bool txn_memtable_branch_named(HostSharedRecord record) { return txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_CHUNK_WORD_INDEX) != 0 || txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_IDENTITY_WORD) != 0 || txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_INDEX) != 0 || - txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_PENDING_INDEX) != 0; + txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_PENDING_INDEX) != 0 || + txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_INDEX) != 0 || + txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_PENDING_INDEX) != 0; } /* @@ -739,9 +740,9 @@ txn_memtable_branch_BranchDeltas_Pool_reclaim(HostIoIo *io, bool *result, HostEr return frame.ok; } -/* TXN_MEMTABLE_BRANCH_BRANCH_DELTAS_POOL_OPS is BranchDeltas' Shape and Pool: a working state's delta, two locks, and seven atomics. */ +/* TXN_MEMTABLE_BRANCH_BRANCH_DELTAS_POOL_OPS is BranchDeltas' Shape and Pool: a working state's delta, three locks, and twelve atomics. */ const HostSharedPoolOps TXN_MEMTABLE_BRANCH_BRANCH_DELTAS_POOL_OPS = { - .shape = {.id = UINT64CONST(0x74786e000014), .locks = 2, .atomics = 7}, + .shape = {.id = UINT64CONST(0x74786e000014), .locks = 3, .atomics = 12}, .name = {"working state's delta", 21}, .count = txn_memtable_branch_BranchDeltas_Pool_count, .reclaim = txn_memtable_branch_BranchDeltas_Pool_reclaim, @@ -948,7 +949,9 @@ txn_memtable_branch_filled_body_0(void *frame_) frame->held_live = true; record = host_shared_Held_record(&frame->held); named = txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_CHUNK_WORD_INDEX) != 0 || - txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_IDENTITY_WORD) != 0; + txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_IDENTITY_WORD) != 0 || + txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_INDEX) != 0 || + txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_PENDING_INDEX) != 0; if (found->f1 != 0 && found->f1 <= PG_UINT32_MAX && named && txn_memtable_branch_load(record, TXN_MEMTABLE_BRANCH_REPOSITORY) == found->f1) { @@ -972,9 +975,9 @@ txn_memtable_branch_filled_body_0(void *frame_) /* * txn_memtable_branch_filled returns each repository whose working state's - * delta names a memtable, from a sweep of the pool, whose versions a - * checkpoint saves in its runs whether or not a storage of the repository - * has a state. + * delta names a memtable or an identity spill (bugs row 291), from a sweep + * of the pool, whose versions a checkpoint saves in its runs whether or not a + * storage of the repository has a state. */ bool txn_memtable_branch_filled(HostIoIo *io, StdVec *result, HostError *error) @@ -1116,7 +1119,7 @@ txn_memtable_branch_prepare(HostIoIo *io, HostSharedRecord deltas, const uint8 * TxnMemtableTableReservation unstored; return txn_memtable_table_prepared(io, &aim, 1, false, none, inside, inside_captures, inside_drop, &has_unstored, - &unstored, error); + &unstored, NULL, error); } /* @@ -1135,12 +1138,14 @@ txn_memtable_branch_finish(HostIoIo *io, TxnMemtableTablePreparedBranchDeltas *p TxnMemtableVersionChange change = {.writer = writer, .op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_KEY_SHARE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}, .has_lsn = has_lsn, .lsn = lsn}; bool pushed; + TxnStatusRoomed unroomed = {0}; op.u.put.value = *chunk; std_vec_new(chunk, 1, 1); std_vec_new(&op.u.put.addresses, sizeof(chunks_TupleU32Locator_t), _Alignof(chunks_TupleU32Locator_t)); std_vec_new(&op.u.put.values, sizeof(uint32), _Alignof(uint32)); - pushed = txn_memtable_table_pushed(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, &change, result, error); + pushed = txn_memtable_table_pushed(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, &change, false, + unroomed, result, error); *chunk = op.u.put.value; return pushed; } @@ -2548,7 +2553,7 @@ test_txn_memtable_branch_tests_an_abort_leaves_the_chunk_a_committed_transaction host_Error_Debug_fmt); txn_memtable_branch_tests_chunks(812, 68, io, deltas, 4, txn_memtable_branch_tests_wrote(aborted), false, 0); simc_abort(aborted); - if (!txn_written_aborted(io, &error)) + if (!txn_written_aborted(io, 0, &error)) STD_RESULT_EXPECT_FAILED(814, 37, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); txn_memtable_branch_tests_got_is(815, 9, 815, 58, io, deltas, 4, "chunk"); diff --git a/src/doltgres/txn/c/memtable/index.c b/src/doltgres/txn/c/memtable/index.c index 3de461d94f41c..03303024d9e20 100644 --- a/src/doltgres/txn/c/memtable/index.c +++ b/src/doltgres/txn/c/memtable/index.c @@ -47,9 +47,6 @@ /* TXN_MEMTABLE_INDEX_LEVELS is LEVELS, the first bit of a node's height in its place word, above its key's length. */ #define TXN_MEMTABLE_INDEX_LEVELS 56 -/* TXN_MEMTABLE_INDEX_WORDS is WORDS, the count of a node's words before its links: its head and its place word. */ -#define TXN_MEMTABLE_INDEX_WORDS UINT64CONST(2) - /* A range's pages are at most the delta's slots, so a page's index fits a node's name above its word. */ StaticAssertDecl(HOST_DELTA_PAGE_SLOTS <= UINT64CONST(1) << 32, "a page's index fits a node's name"); @@ -154,7 +151,7 @@ HostDeltaPageSize txn_memtable_index_least(const uint8 *key, size_t key_len) { HostDeltaPageSize size = {(uint32) key_len, - (uint32) (1 + TXN_MEMTABLE_INDEX_WORDS + TXN_MEMTABLE_INDEX_HEIGHT)}; + (uint32) TXN_MEMTABLE_INDEX_LEAST_WORDS}; return size; } @@ -258,20 +255,17 @@ txn_memtable_index_Place_loaded(const TxnMemtableIndexPlace *self, uint64 level, } /* - * txn_memtable_index_key_cmp compares a key with a node's key, lent and - * given back at once, answering false on an error. + * txn_memtable_index_Place_cmp compares a key with the key of the node at an + * opened place, lent and given back at once, answering false on an error. */ static bool -txn_memtable_index_key_cmp(const TxnMemtableIndexKeyedOps *keyed, const HostSharedGuard *guard, - const HostDeltaHold *hold, TxnMemtableIndexNode node, const uint8 *key, size_t key_len, - StdOrdering *result, HostError *error) +txn_memtable_index_Place_cmp(const TxnMemtableIndexPlace *self, const uint8 *key, size_t key_len, + StdOrdering *result, HostError *error) { - TxnMemtableIndexPlace place; HostDeltaLent lent; StdBytes bytes; - if (!txn_memtable_index_Place_at(&keyed->closes, guard, hold, node, &place, error) || - !txn_memtable_index_Place_key(&place, &lent, error)) + if (!txn_memtable_index_Place_key(self, &lent, error)) return false; bytes = host_delta_Lent_Deref_deref(&lent); *result = std_bytes_Ord_cmp(key, key_len, bytes.ptr, bytes.len); @@ -279,6 +273,38 @@ txn_memtable_index_key_cmp(const TxnMemtableIndexKeyedOps *keyed, const HostShar return true; } +/* + * txn_memtable_index_Place_next loads the link at a level of the node at an + * opened place afresh, answering false on an error and setting some when it + * names a node. + */ +static bool +txn_memtable_index_Place_next(const TxnMemtableIndexPlace *self, uint64 level, bool *some, + TxnMemtableIndexNode *result, HostError *error) +{ + uint64 link; + + if (!txn_memtable_index_Place_loaded(self, level, &link, error)) + return false; + *some = txn_memtable_index_Node_named(link, result); + return true; +} + +/* + * txn_memtable_index_key_cmp compares a key with a node's key, lent and + * given back at once, answering false on an error. + */ +static bool +txn_memtable_index_key_cmp(const TxnMemtableIndexKeyedOps *keyed, const HostSharedGuard *guard, + const HostDeltaHold *hold, TxnMemtableIndexNode node, const uint8 *key, size_t key_len, + StdOrdering *result, HostError *error) +{ + TxnMemtableIndexPlace place; + + return txn_memtable_index_Place_at(&keyed->closes, guard, hold, node, &place, error) && + txn_memtable_index_Place_cmp(&place, key, key_len, result, error); +} + /* * txn_memtable_index_next_at loads the link at a level of the node at a * place, answering false on an error and setting some when it names a node. @@ -289,13 +315,9 @@ txn_memtable_index_next_at(const TxnMemtableIndexKeyedOps *keyed, const HostShar TxnMemtableIndexNode *result, HostError *error) { TxnMemtableIndexPlace place; - uint64 link; - if (!txn_memtable_index_Place_at(&keyed->closes, guard, hold, node, &place, error) || - !txn_memtable_index_Place_loaded(&place, level, &link, error)) - return false; - *some = txn_memtable_index_Node_named(link, result); - return true; + return txn_memtable_index_Place_at(&keyed->closes, guard, hold, node, &place, error) && + txn_memtable_index_Place_next(&place, level, some, result, error); } /* txn_memtable_index_first returns the index's first node, or nothing while it holds no key. */ @@ -516,7 +538,12 @@ typedef struct TxnMemtableIndexPath /* * txn_memtable_index_path searches the index for a key from its first node, * at each level from the highest down following links while the next node's - * key is below the key. + * key is below the key. It opens each node it reaches once and keeps its + * place, a page the hold keeps open, loading each link from it afresh, so a + * link stored since is seen as a reopen would see it (bugs row 355). A node + * on the page of the node before it takes that page, since its caller holds + * the range's lock through the walk and the range's list changes its entries + * only under that lock held exclusively. */ static bool txn_memtable_index_path(const TxnMemtableIndexKeyedOps *keyed, const HostSharedGuard *guard, @@ -525,13 +552,15 @@ txn_memtable_index_path(const TxnMemtableIndexKeyedOps *keyed, const HostSharedG { TxnMemtableIndexNode first; TxnMemtableIndexNode at; + TxnMemtableIndexPlace at_place; StdOrdering order; result->has_before = false; result->has_at = false; if (!txn_memtable_index_first(keyed, guard, &first)) return true; - if (!txn_memtable_index_key_cmp(keyed, guard, hold, first, key, key_len, &order, error)) + if (!txn_memtable_index_Place_at(&keyed->closes, guard, hold, first, &at_place, error) || + !txn_memtable_index_Place_cmp(&at_place, key, key_len, &order, error)) return false; if (order != STD_ORDERING_GREATER) { @@ -546,21 +575,30 @@ txn_memtable_index_path(const TxnMemtableIndexKeyedOps *keyed, const HostSharedG { bool some; TxnMemtableIndexNode next; + TxnMemtableIndexPlace next_place; - if (!txn_memtable_index_next_at(keyed, guard, hold, at, level, &some, &next, error)) + if (!txn_memtable_index_Place_next(&at_place, level, &some, &next, error)) return false; if (!some) break; - if (!txn_memtable_index_key_cmp(keyed, guard, hold, next, key, key_len, &order, error)) + if (txn_memtable_index_Node_page(next) == txn_memtable_index_Node_page(at)) + { + next_place.page = at_place.page; + next_place.word = txn_memtable_index_Node_word(next); + } + else if (!txn_memtable_index_Place_at(&keyed->closes, guard, hold, next, &next_place, error)) + return false; + if (!txn_memtable_index_Place_cmp(&next_place, key, key_len, &order, error)) return false; if (order != STD_ORDERING_GREATER) break; at = next; + at_place = next_place; } result->before[level] = at; } result->has_before = true; - return txn_memtable_index_next_at(keyed, guard, hold, at, 0, &result->has_at, &result->at, error); + return txn_memtable_index_Place_next(&at_place, 0, &result->has_at, &result->at, error); } /* @@ -2142,7 +2180,8 @@ test_txn_memtable_index_tests_a_new_keys_version_and_node_go_together_on_a_fresh STD_OPTION_UNWRAP_FAILED(693, 45); std_vec_new(&spare, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); std_vec_new(&buffer, 1, 1); - if (!txn_memtable_version_push(&open, &hold, c, sizeof(c), &change, &has_fresh, &fresh, &spare, &buffer, &wanted, + if (!txn_memtable_version_push(&open, &hold, c, sizeof(c), &change, false, &has_fresh, &fresh, &spare, &buffer, + &wanted, &error)) STD_RESULT_EXPECT_FAILED(710, 21, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); diff --git a/src/doltgres/txn/c/memtable/map.c b/src/doltgres/txn/c/memtable/map.c index 4f1f2b2242e2b..7407c42f6f1e7 100644 --- a/src/doltgres/txn/c/memtable/map.c +++ b/src/doltgres/txn/c/memtable/map.c @@ -120,20 +120,29 @@ txn_memtable_map_Entries_next(TxnMemtableVersionChain *chain, bool *some, TxnMem } } +/* txn_memtable_map_newest_of returns a key's node's newest version in a map's range that isn't a values mark. */ +static bool +txn_memtable_map_newest_of(const HostSharedGuard *guard, const HostDeltaHold *hold, TxnMemtableIndexNode node, + bool *some, TxnMemtableVersionVersion *result, HostError *error) +{ + TxnMemtableVersionChain chain; + + return txn_memtable_version_chain(&TXN_MEMTABLE_RANGE_RANGE_KEYED_OPS, guard, hold, node, &chain, error) && + txn_memtable_map_Entries_next(&chain, some, result, error); +} + /* txn_memtable_map_newest_in returns a key's newest version in a map's range that isn't a values mark, under its lock either way. */ static bool txn_memtable_map_newest_in(const HostSharedGuard *guard, const HostDeltaHold *hold, const uint8 *key, size_t key_len, bool *some, TxnMemtableVersionVersion *result, HostError *error) { TxnMemtableIndexNode node; - TxnMemtableVersionChain chain; if (!txn_memtable_index_find(&TXN_MEMTABLE_RANGE_RANGE_KEYED_OPS, guard, hold, key, key_len, some, &node, error)) return false; if (!*some) return true; - return txn_memtable_version_chain(&TXN_MEMTABLE_RANGE_RANGE_KEYED_OPS, guard, hold, node, &chain, error) && - txn_memtable_map_Entries_next(&chain, some, result, error); + return txn_memtable_map_newest_of(guard, hold, node, some, result, error); } /* txn_memtable_map_valued returns the value a map's version puts, or none for a removal. */ @@ -377,12 +386,12 @@ txn_memtable_map_newest_reading(void *captures, const HostDeltaHold *hold, TxnMe } /* - * txn_memtable_map_newest returns a key's newest entry in a storage's map - * outside any window: nothing for a key with none, and none for a removal. + * txn_memtable_map_in_memtable returns a key's newest entry in a storage's + * map's memtable outside any window, as newest does, its spills unread. */ bool -txn_memtable_map_newest(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 key, bool *some, - TxnMemtableMapEntry *result, HostError *error) +txn_memtable_map_in_memtable(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 key, bool *some, + TxnMemtableMapEntry *result, HostError *error) { uint8 bytes[TXN_MEMTABLE_MAP_KEY]; txn_memtable_map_newest_reading_t reading = {bytes}; @@ -399,6 +408,86 @@ txn_memtable_map_newest(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap return true; } +/* txn_memtable_map_entry reports whether a map's version is an entry's, a put or a removal, and not a values mark. */ +static bool +txn_memtable_map_entry(const TxnMemtableVersionVersion *version) +{ + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); + + return txn_memtable_version_Record_op(&record).tag != TXN_MEMTABLE_VERSION_OP_REF_NAMED; +} + +/* + * txn_memtable_map_spilled returns a block's free space entry's versions in a + * storage's spills, newest first, read after its chain in the memtable: a + * spill takes only keys whose every version is its transaction's, so + * whatever the memtable holds of a spilled key came after. + */ +static bool +txn_memtable_map_spilled(HostIoIo *io, HostSharedRecord state, const uint8 *key, uint64 *listed, StdVec *result, + HostError *error) +{ + HostDeltaHold hold; + bool ok; + + if (!host_delta_hold(io, &hold, error)) + return false; + ok = txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, io, &hold, state, TXN_RUNS_KIND_SPILLED_FREE, key, + TXN_MEMTABLE_MAP_KEY, listed, result, error); + host_delta_Hold_drop(&hold); + return ok; +} + +/* + * txn_memtable_map_spilled_newest returns a block's newest free space entry + * in a storage's spills, the entry below its memtable's chain, a values mark + * passed over as newest_in passes over one: nothing when they hold no version + * of it, and none for a removal. + */ +bool +txn_memtable_map_spilled_newest(HostIoIo *io, HostSharedRecord state, uint32 key, bool *some, + TxnMemtableMapEntry *result, HostError *error) +{ + uint8 bytes[TXN_MEMTABLE_MAP_KEY]; + StdVec versions; + bool ok = true; + + *some = false; + std_u32_to_be_bytes(key, bytes); + if (!txn_memtable_map_spilled(io, state, bytes, NULL, &versions, error)) + return false; + for (size_t k = 0; k < versions.len && !*some && ok; k++) + { + const TxnMemtableVersionVersion *version = &STD_VEC_AT(&versions, TxnMemtableVersionVersion, k); + StdBytes value; + + if (!txn_memtable_map_entry(version)) + continue; + ok = txn_memtable_map_valued(version, &result->has_value, &value, error); + if (ok && result->has_value) + std_slice_to_vec_Checked(&result->value, value.ptr, value.len, 1, 1); + *some = ok; + } + txn_memtable_version_versions_drop(&versions, 0); + return ok; +} + +/* + * txn_memtable_map_newest returns a key's newest entry in a storage's map + * outside any window: nothing for a key with none, and none for a removal. A + * free space entry with no version in the memtable is read in the storage's + * spills. + */ +bool +txn_memtable_map_newest(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 key, bool *some, + TxnMemtableMapEntry *result, HostError *error) +{ + if (!txn_memtable_map_in_memtable(io, state, map, key, some, result, error)) + return false; + return *some || map != TXN_MEMTABLE_MAP_MAP_FREE || + txn_memtable_map_spilled_newest(io, state, key, some, result, error); +} + /* txn_memtable_map_taken returns the tuple id a free space entry's version took, or names as its take's values mark, or 0 for none. */ HostRow txn_memtable_map_taken(TxnMemtableVersionOpRef op) @@ -459,7 +548,7 @@ typedef struct txn_memtable_map_takers_reading_t } txn_memtable_map_takers_reading_t; /* txn_memtable_map_taker reads one version as a taker, answering false on an error. */ -static bool +bool txn_memtable_map_taker(const TxnMemtableVersionVersion *version, txn_TupleWriterRowEntryU64TupleVecU32Bool_t *taker, HostError *error) { @@ -576,12 +665,10 @@ txn_memtable_map_takers_reading_body_0(void *frame_) _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); frame->taker_live = false; } - std_slice_reverse(reading->takers.ptr, reading->takers.len, sizeof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t), - _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); frame->ok = true; } -/* txn_memtable_map_takers_reading reads a key's chain as takers, oldest first, under its range's shared lock. */ +/* txn_memtable_map_takers_reading reads a key's chain as takers, newest first, under its range's shared lock. */ static bool txn_memtable_map_takers_reading(void *captures, const HostDeltaHold *hold, TxnMemtableTableAt at, const HostSharedGuard *guard, HostError *error) @@ -620,24 +707,70 @@ txn_memtable_map_takers_pad_0(void *frame_) } } -/* txn_memtable_map_takers_body_0 is takers' body. */ -static void -txn_memtable_map_takers_body_0(void *frame_) +/* txn_memtable_map_takers_read is takers' read: the key's chain in the memtable, then a free space entry's spilled versions below it. */ +static bool +txn_memtable_map_takers_read(void *captures, uint64 *listed, HostError *error) { - txn_memtable_map_takers_frame_0_t *frame = frame_; + txn_memtable_map_takers_frame_0_t *frame = captures; bool some; - frame->reading.key = frame->bytes; std_vec_new(&frame->reading.takers, sizeof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t), _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); frame->takers_live = true; if (!txn_memtable_table_read(txn_memtable_map_tree(frame->map), frame->io, frame->state, frame->bytes, sizeof(frame->bytes), txn_memtable_map_takers_reading, &frame->reading, &some, - frame->error)) + error)) { txn_memtable_map_takers_pad_0(frame); - return; + return false; + } + if (frame->map == TXN_MEMTABLE_MAP_MAP_FREE) + { + StdVec spilled; + bool ok; + + if (!txn_memtable_map_spilled(frame->io, frame->state, frame->bytes, listed, &spilled, error)) + { + txn_memtable_map_takers_pad_0(frame); + return false; + } + ok = std_vec_try_reserve(&frame->reading.takers, spilled.len, + sizeof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t), + _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); + if (!ok) + *error = host_Error_out_of_memory(); + for (size_t k = 0; k < spilled.len && ok; k++) + { + txn_TupleWriterRowEntryU64TupleVecU32Bool_t taker; + + ok = txn_memtable_map_taker(&STD_VEC_AT(&spilled, TxnMemtableVersionVersion, k), &taker, error); + if (ok) + std_vec_push_Checked(&frame->reading.takers, &taker, sizeof(taker), + _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); + } + txn_memtable_version_versions_drop(&spilled, 0); + if (!ok) + { + txn_memtable_map_takers_pad_0(frame); + return false; + } } + return true; +} + +/* txn_memtable_map_takers_body_0 is takers' body. */ +static void +txn_memtable_map_takers_body_0(void *frame_) +{ + txn_memtable_map_takers_frame_0_t *frame = frame_; + + frame->reading.key = frame->bytes; + if (!txn_spill_taken(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->state, txn_memtable_map_takers_read, + txn_memtable_map_takers_pad_0, frame, frame->error)) + return; + std_slice_reverse(frame->reading.takers.ptr, frame->reading.takers.len, + sizeof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t), + _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); *frame->result = frame->reading.takers; frame->takers_live = false; frame->ok = true; @@ -647,7 +780,8 @@ txn_memtable_map_takers_body_0(void *frame_) * txn_memtable_map_takers returns a key's versions in a storage's map as * VACUUM reads them, oldest first: each one's writer, the tuple id its take * took or names, its entry, its ender widened to a full xid, and its row's - * value ids with whether it is a values mark. + * value ids with whether it is a values mark. A free space entry's spilled + * versions come below its memtable's. */ bool txn_memtable_map_takers(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 key, StdVec *result, @@ -661,18 +795,72 @@ txn_memtable_map_takers(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap return frame.ok; } -/* txn_memtable_map_rows_reading_t is a free space map reading's captures: the key, the tuple id, the ender or the LSN, and the answer. */ +/* + * txn_memtable_map_spilled_takes returns the tuple ids a block's free space + * entry's spilled versions took, which VACUUM frees only once a fold took + * their file and its base holds them, since a spilled version holds no + * ender. + */ +bool +txn_memtable_map_spilled_takes(HostIoIo *io, HostSharedRecord state, uint32 block, StdVec *result, HostError *error) +{ + uint8 bytes[TXN_MEMTABLE_MAP_KEY]; + StdVec versions; + bool ok = true; + + std_u32_to_be_bytes(block, bytes); + if (!txn_memtable_map_spilled(io, state, bytes, NULL, &versions, error)) + return false; + std_vec_new(result, sizeof(HostRow), _Alignof(HostRow)); + for (size_t k = 0; k < versions.len && ok; k++) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(&versions, + TxnMemtableVersionVersion, + k)); + HostRow tid = txn_memtable_map_taken(txn_memtable_version_Record_op(&record)); + + if (tid.f0 == 0) + continue; + ok = std_vec_try_reserve(result, 1, sizeof(tid), _Alignof(HostRow)); + if (ok) + std_vec_push_Checked(result, &tid, sizeof(tid), _Alignof(HostRow)); + else + *error = host_Error_out_of_memory(); + } + txn_memtable_version_versions_drop(&versions, 0); + if (!ok) + std_vec_free_Checked(result, sizeof(HostRow), _Alignof(HostRow)); + return ok; +} + +/* txn_memtable_map_marker answers the tuple id a free space entry's version holds when it carries the end of a WAL record. */ +static bool +txn_memtable_map_marker(const TxnMemtableVersionVersion *version, uint64 lsn, HostRow *result) +{ + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); + TxnMemtableVersionOpRef op = txn_memtable_version_Record_op(&record); + uint64 at; + + if (op.tag != TXN_MEMTABLE_VERSION_OP_REF_PUT || !txn_memtable_version_Record_lsn(&record, &at) || at != lsn || + op.u.put.tuple.f0 == 0) + return false; + *result = op.u.put.tuple; + return true; +} + +/* txn_memtable_map_rows_reading_t is a free space map reading's captures: the key, the tuple id, the ender or the LSN, the take's writer, and the answer. */ typedef struct txn_memtable_map_rows_reading_t { const uint8 *key; HostRow tid; uint32 ender; + uint64 writer; uint64 lsn; bool found; StdVec tids; } txn_memtable_map_rows_reading_t; -/* txn_memtable_map_ended_reading stores the ender in the version of a free space entry that took the tuple id. */ +/* txn_memtable_map_ended_reading stores the ender in the version of a free space entry by which the writer took the tuple id. */ static bool txn_memtable_map_ended_reading(void *captures, const HostDeltaHold *hold, TxnMemtableTableAt at, const HostSharedGuard *guard, HostError *error) @@ -700,7 +888,8 @@ txn_memtable_map_ended_reading(void *captures, const HostDeltaHold *hold, TxnMem if (!some) return true; record = txn_memtable_version_Version_record(&version); - taken = txn_memtable_map_taken(txn_memtable_version_Record_op(&record)).f0 == reading->tid.f0; + taken = txn_memtable_map_taken(txn_memtable_version_Record_op(&record)).f0 == reading->tid.f0 && + txn_memtable_version_Record_writer(&record).xid == reading->writer; if (taken) txn_memtable_version_Version_end(&version, reading->ender); txn_memtable_version_Version_drop(&version); @@ -711,13 +900,16 @@ txn_memtable_map_ended_reading(void *captures, const HostDeltaHold *hold, TxnMem /* * txn_memtable_map_ended stores a fold's drop's ender in the free space - * map's version that took a tuple id, so VACUUM reads the row's end. + * map's version by which the row's writer took its tuple id, so VACUUM reads + * the row's end; a standby whose fold lags may already hold a later take of + * the tuple id, which keeps no ender. */ bool -txn_memtable_map_ended(HostIoIo *io, HostSharedRecord state, HostRow tid, uint32 ender, HostError *error) +txn_memtable_map_ended(HostIoIo *io, HostSharedRecord state, HostRow tid, uint64 writer, uint32 ender, + HostError *error) { uint8 bytes[TXN_MEMTABLE_MAP_KEY]; - txn_memtable_map_rows_reading_t reading = {bytes, tid, ender}; + txn_memtable_map_rows_reading_t reading = {.key = bytes, .tid = tid, .ender = ender, .writer = writer}; bool some; std_u32_to_be_bytes((uint32) STD_SHR_FOLDED(u64, tid.f0, 16), bytes); @@ -745,44 +937,87 @@ txn_memtable_map_marked_reading(void *captures, const HostDeltaHold *hold, TxnMe for (;;) { TxnMemtableVersionVersion version; - TxnMemtableVersionRecord record; - TxnMemtableVersionOpRef op; - uint64 at_lsn; + HostRow tid; if (!txn_memtable_map_Entries_next(&chain, &some, &version, error)) return false; if (!some) return true; - record = txn_memtable_version_Version_record(&version); - op = txn_memtable_version_Record_op(&record); - if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_PUT && txn_memtable_version_Record_lsn(&record, &at_lsn) && - at_lsn == reading->lsn && op.u.put.tuple.f0 != 0) - std_vec_push_Checked(&reading->tids, &op.u.put.tuple, sizeof(HostRow), _Alignof(HostRow)); + if (txn_memtable_map_marker(&version, reading->lsn, &tid)) + std_vec_push_Checked(&reading->tids, &tid, sizeof(HostRow), _Alignof(HostRow)); txn_memtable_version_Version_drop(&version); } } +/* txn_memtable_map_marked_read_t is marked's read's captures: the state and the walk's reading. */ +typedef struct txn_memtable_map_marked_read_t +{ + HostIoIo *io; + HostSharedRecord state; + txn_memtable_map_rows_reading_t reading; +} txn_memtable_map_marked_read_t; + +/* txn_memtable_map_marked_dropped drops the tuple ids a read of marked gave. */ +static void +txn_memtable_map_marked_dropped(void *captures) +{ + txn_memtable_map_marked_read_t *c = captures; + + std_vec_free_Checked(&c->reading.tids, sizeof(HostRow), _Alignof(HostRow)); +} + +/* txn_memtable_map_marked_read is marked's read: the block's chain in the memtable, then its spilled versions. */ +static bool +txn_memtable_map_marked_read(void *captures, uint64 *listed, HostError *error) +{ + txn_memtable_map_marked_read_t *c = captures; + bool some; + StdVec spilled; + bool ok = true; + + std_vec_new(&c->reading.tids, sizeof(HostRow), _Alignof(HostRow)); + if (!txn_memtable_table_read(&TXN_MEMTABLE_MAP_FREE_TREE_OPS, c->io, c->state, c->reading.key, + TXN_MEMTABLE_MAP_KEY, txn_memtable_map_marked_reading, &c->reading, &some, error) || + !txn_memtable_map_spilled(c->io, c->state, c->reading.key, listed, &spilled, error)) + { + txn_memtable_map_marked_dropped(c); + return false; + } + for (size_t k = 0; k < spilled.len && ok; k++) + { + HostRow tid; + + if (!txn_memtable_map_marker(&STD_VEC_AT(&spilled, TxnMemtableVersionVersion, k), c->reading.lsn, &tid)) + continue; + ok = std_vec_try_reserve(&c->reading.tids, 1, sizeof(tid), _Alignof(HostRow)); + if (ok) + std_vec_push_Checked(&c->reading.tids, &tid, sizeof(tid), _Alignof(HostRow)); + else + *error = host_Error_out_of_memory(); + } + txn_memtable_version_versions_drop(&spilled, 0); + if (!ok) + txn_memtable_map_marked_dropped(c); + return ok; +} + /* * txn_memtable_map_marked returns the tuple ids a block's free space map - * entries put with an LSN, the takes a row change record's redo made. + * entries put with an LSN, the takes a row change record's redo made, in its + * memtable and then its spills. */ bool txn_memtable_map_marked(HostIoIo *io, HostSharedRecord state, uint32 block, uint64 lsn, StdVec *result, HostError *error) { uint8 bytes[TXN_MEMTABLE_MAP_KEY]; - txn_memtable_map_rows_reading_t reading = {.key = bytes, .lsn = lsn}; - bool some; + txn_memtable_map_marked_read_t captures = {io, state, {.key = bytes, .lsn = lsn}}; std_u32_to_be_bytes(block, bytes); - std_vec_new(&reading.tids, sizeof(HostRow), _Alignof(HostRow)); - if (!txn_memtable_table_read(&TXN_MEMTABLE_MAP_FREE_TREE_OPS, io, state, bytes, sizeof(bytes), - txn_memtable_map_marked_reading, &reading, &some, error)) - { - std_vec_free_Checked(&reading.tids, sizeof(HostRow), _Alignof(HostRow)); + if (!txn_spill_taken(&TXN_SPILL_STATE_HOME_OPS, io, state, txn_memtable_map_marked_read, + txn_memtable_map_marked_dropped, &captures, error)) return false; - } - *result = reading.tids; + *result = captures.reading.tids; return true; } @@ -847,21 +1082,22 @@ txn_memtable_map_marks(HostIoIo *io, HostSharedRecord state, uint32 block, HostR } /* - * txn_memtable_map_holds reports whether a storage's map, in its memtable - * or, for the value map, its spills, holds the version a record of an LSN - * pushed, so its redo skips the push. + * txn_memtable_map_holds reports whether a storage's map, in its memtable or + * its spills, holds the version a record of an LSN pushed, so its redo skips + * the push. */ bool txn_memtable_map_holds(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 key, uint64 lsn, bool *result, HostError *error) { uint8 bytes[TXN_MEMTABLE_MAP_KEY]; + TxnRunsKind kind = map == TXN_MEMTABLE_MAP_MAP_FREE ? TXN_RUNS_KIND_SPILLED_FREE : TXN_RUNS_KIND_SPILLED_VALUES; std_u32_to_be_bytes(key, bytes); if (!txn_memtable_table_held(txn_memtable_map_tree(map), io, state, bytes, sizeof(bytes), lsn, result, error)) return false; - return map == TXN_MEMTABLE_MAP_MAP_FREE || *result || - txn_spill_holds(io, state, TXN_RUNS_KIND_SPILLED_VALUES, bytes, sizeof(bytes), lsn, result, error); + return *result || txn_spill_holds(&TXN_SPILL_STATE_HOME_OPS, io, state, kind, bytes, sizeof(bytes), lsn, result, + error); } /* txn_memtable_map_cut_reading_t is cut's reading's captures: the state and the cut read. */ @@ -1116,13 +1352,13 @@ txn_memtable_map_seek_body_0(void *frame_) } /* - * txn_memtable_map_seek returns the first key at or past a key in a - * storage's map with an entry, and that entry, walking step by step, or - * nothing past the last. + * txn_memtable_map_sought returns the first key at or past a key in a + * storage's map's memtable with an entry, and that entry, walking step by + * step, or nothing past the last. */ -bool -txn_memtable_map_seek(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 from, bool *some, - txn_TupleU32Entry_t *result, HostError *error) +static bool +txn_memtable_map_sought(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 from, bool *some, + txn_TupleU32Entry_t *result, HostError *error) { txn_memtable_map_seek_frame_0_t frame = {.io = io, .state = state, .map = map, .from = from, .some = some, .result = result, .error = error}; @@ -1131,6 +1367,68 @@ txn_memtable_map_seek(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap ma return frame.ok; } +/* + * txn_memtable_map_seek returns the lowest key at or past a key with an + * entry in a storage's map, in its memtable or, for the free space map, its + * spills, with its newest value or none for a removal, so that allocation's + * merge masks the base's entry at every block the delta has one for; a key + * both hold takes the memtable's, which came after. + */ +bool +txn_memtable_map_seek(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 from, bool *some, + txn_TupleU32Entry_t *result, HostError *error) +{ + uint8 after[TXN_MEMTABLE_MAP_KEY]; + HostDeltaHold hold; + bool next; + txn_TupleVecU8VecVersion_t spilled; + const TxnMemtableVersionVersion *version = NULL; + uint32 block; + bool ok; + + if (!txn_memtable_map_sought(io, state, map, from, some, result, error)) + return false; + if (map != TXN_MEMTABLE_MAP_MAP_FREE) + return true; + if (from > 0) + std_u32_to_be_bytes(from - 1, after); + if (!host_delta_hold(io, &hold, error)) + return false; + ok = txn_spill_next(&TXN_SPILL_STATE_HOME_OPS, io, &hold, state, TXN_RUNS_KIND_SPILLED_FREE, from > 0, after, + sizeof(after), TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, NULL, &next, &spilled, error); + host_delta_Hold_drop(&hold); + if (!ok || !next) + return ok; + if (spilled.f0.len != TXN_MEMTABLE_MAP_KEY) + { + *error = txn_memtable_map_internal("a map's key isn't 4 bytes"); + ok = false; + } + block = ok ? std_u32_from_be_bytes(spilled.f0.ptr) : 0; + for (size_t k = 0; ok && k < spilled.f1.len && version == NULL; k++) + if (txn_memtable_map_entry(&STD_VEC_AT(&spilled.f1, TxnMemtableVersionVersion, k))) + version = &STD_VEC_AT(&spilled.f1, TxnMemtableVersionVersion, k); + if (ok && version != NULL && !(*some && result->f0 <= block)) + { + StdBytes value; + txn_TupleU32Entry_t below = {block}; + + ok = txn_memtable_map_valued(version, &below.f1.has_value, &value, error); + if (ok) + { + if (below.f1.has_value) + std_slice_to_vec_Checked(&below.f1.value, value.ptr, value.len, 1, 1); + if (*some && result->f1.has_value) + std_vec_free_Checked(&result->f1.value, 1, 1); + *result = below; + *some = true; + } + } + std_vec_free_Checked(&spilled.f0, 1, 1); + txn_memtable_version_versions_drop(&spilled.f1, 0); + return ok; +} + /* txn_memtable_map_named_t is named's step's captures: the change's writer, its LSN, the row's tuple id, and its value ids. */ typedef struct txn_memtable_map_named_t { @@ -1164,16 +1462,17 @@ txn_memtable_map_named_step(void *captures, TxnMemtableIndexKeyedOpen *open, con /* * txn_memtable_map_named pushes a take's values mark inside the window: the * row's tuple id and its value ids, under the take's key, changing no entry, - * taking the value ids. + * taking the value ids, and counts it at its writer's entry when it names + * one. */ bool txn_memtable_map_named(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t take, - TxnMemtableVersionWriter writer, bool has_lsn, uint64 lsn, HostRow tuple, StdVec *values, - HostError *error) + TxnMemtableVersionWriter writer, bool has_roomed, TxnStatusRoomed roomed, bool has_lsn, + uint64 lsn, HostRow tuple, StdVec *values, HostError *error) { txn_memtable_map_named_t named = {writer, has_lsn, lsn, tuple, values}; bool ok = txn_memtable_table_within(io, prepared, txn_memtable_table_Prepared_target(prepared, take), - txn_memtable_map_named_step, &named, error); + has_roomed, roomed, txn_memtable_map_named_step, &named, error); std_vec_free_Checked(values, sizeof(uint32), _Alignof(uint32)); return ok; @@ -1193,6 +1492,7 @@ typedef struct txn_memtable_map_took_t bool has_base; const uint8 *base; size_t base_len; + const TxnMemtableMapRead *read; TxnMemtableMapStep (*step) (void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out); HostRow (*tuple) (void *captures); @@ -1201,10 +1501,10 @@ typedef struct txn_memtable_map_took_t TxnMemtableMapTaken *result; } txn_memtable_map_took_t; -/* txn_memtable_map_took_pushed pushes the step's put or removal with the reserve and runs then, answering false on an error. */ +/* txn_memtable_map_took_pushed pushes the step's put or removal with the reserve at the key's Found and runs then, answering false on an error. */ static bool txn_memtable_map_took_pushed(txn_memtable_map_took_t *took, TxnMemtableIndexKeyedOpen *open, - const HostDeltaHold *hold, const uint8 *key, size_t key_len, + const HostDeltaHold *hold, TxnMemtableIndexFound found, const uint8 *key, size_t key_len, TxnMemtableTableReserve *reserve, TxnMemtableMapStep answer, StdVec *value, bool has_was, uint8 was, HostError *error) { @@ -1229,7 +1529,7 @@ txn_memtable_map_took_pushed(txn_memtable_map_took_t *took, TxnMemtableIndexKeye op.tag = tuple.f0 == 0 ? TXN_MEMTABLE_OP_DELETE : TXN_MEMTABLE_OP_EMPTIED; op.u.emptied = tuple; } - ok = txn_memtable_table_Reserve_push(reserve, open, hold, key, key_len, &change, &handle, error); + ok = txn_memtable_table_Reserve_push_found(reserve, open, hold, found, key, key_len, &change, &handle, error); if (op.tag == TXN_MEMTABLE_OP_PUT) reserve->value = op.u.put.value; if (!ok) @@ -1243,12 +1543,74 @@ txn_memtable_map_took_pushed(txn_memtable_map_took_t *took, TxnMemtableIndexKeye return true; } -/* txn_memtable_map_took_step is took's step at the open range that holds the take's key. */ +/* txn_memtable_map_unmoved reports whether the entry a take's finish finds and the state's base generation are as its caller read them. */ +static bool +txn_memtable_map_unmoved(const TxnMemtableMapRead *read, HostSharedRecord state, bool has_current, + const uint8 *current, size_t current_len) +{ + if (has_current != read->has_entry) + return false; + if (has_current && (current_len != read->entry_len || + (current_len != 0 && memcmp(current, read->entry, current_len) != 0))) + return false; + return txn_state_generation(state) == read->generation; +} + +/* txn_memtable_map_found_unmoved reports whether a take finds the entry its caller read: the delta's newest when there is one, else the base's value given. */ +static bool +txn_memtable_map_found_unmoved(const TxnMemtableMapRead *read, HostSharedRecord state, bool some, bool has_delta, + StdBytes delta, bool has_base, const uint8 *base, size_t base_len) +{ + return some ? txn_memtable_map_unmoved(read, state, has_delta, delta.ptr, delta.len) : + txn_memtable_map_unmoved(read, state, has_base, base, base_len); +} + +/* + * txn_memtable_map_kept keeps the range of a prepared take's key (table's + * kept) before its caller's window, and reports whether the key's newest + * entry there, else the base's value given, and the state's base generation + * are as its caller read them. While kept the entry stays so through the + * window: no removal or other take reaches the chain, and a fold's install + * puts a key's newest applied version's whole entry in its base unchanged + * (fold.c's ran_key), so a take there needs no read (bugs row 342). + */ +bool +txn_memtable_map_kept(HostIoTakesShort io, struct TxnMemtableTablePreparedState *prepared, size_t take, + HostSharedRecord state, bool has_base, const uint8 *base, size_t base_len, + const TxnMemtableMapRead *read, bool *result, HostError *error) +{ + const TxnMemtableTableAim *aim = &prepared->aims[take]; + bool some; + TxnMemtableVersionVersion newest; + bool has_delta = false; + StdBytes delta = {0}; + bool ok; + + if (!txn_memtable_table_kept(io, prepared, take, error) || + !txn_memtable_map_newest_in(prepared->kept_open.open.guard, &prepared->hold, aim->key.ptr, aim->key.len, + &some, &newest, error)) + return false; + ok = !some || txn_memtable_map_valued(&newest, &has_delta, &delta, error); + if (ok) + *result = txn_memtable_map_found_unmoved(read, state, some, has_delta, delta, has_base, base, base_len); + if (some) + txn_memtable_version_Version_drop(&newest); + return ok; +} + +/* + * txn_memtable_map_took_step is took's step at the open range that holds the + * take's key, which looks the key up once and pushes at what it found: from + * the lookup to the push only reads run under the range's lock, since the + * step callback is given the key's entry and a buffer and nothing that + * reaches the index (bugs row 355). + */ static bool txn_memtable_map_took_step(void *captures, TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, const uint8 *key, size_t key_len, TxnMemtableTableReserve *reserve, HostError *error) { txn_memtable_map_took_t *took = captures; + TxnMemtableIndexFound found; bool some; TxnMemtableVersionVersion newest; bool has_delta = false; @@ -1274,7 +1636,10 @@ txn_memtable_map_took_step(void *captures, TxnMemtableIndexKeyedOpen *open, cons } value = reserve->value; std_vec_new(&reserve->value, 1, 1); - if (!txn_memtable_map_newest_in(open->open.guard, hold, key, key_len, &some, &newest, error)) + some = false; + if (!txn_memtable_index_lookup(open, hold, key, key_len, &found, error) || + (found.tag == TXN_MEMTABLE_INDEX_FOUND_NODE && + !txn_memtable_map_newest_of(open->open.guard, hold, found.u.node, &some, &newest, error))) { reserve->value = value; return false; @@ -1295,6 +1660,16 @@ txn_memtable_map_took_step(void *captures, TxnMemtableIndexKeyedOpen *open, cons reserve->value = value; return false; } + if (took->read != NULL && + !txn_memtable_map_found_unmoved(took->read, took->state, some, has_delta, delta, took->has_base, took->base, + took->base_len)) + { + if (some) + txn_memtable_version_Version_drop(&newest); + reserve->value = value; + *took->result = TXN_MEMTABLE_MAP_TAKEN_MOVED; + return true; + } answer = some ? took->step(took->captures, has_delta, delta.ptr, delta.len, &value) : took->step(took->captures, took->has_base, took->base, took->base_len, &value); if (some) @@ -1306,24 +1681,30 @@ txn_memtable_map_took_step(void *captures, TxnMemtableIndexKeyedOpen *open, cons TXN_MEMTABLE_MAP_TAKEN_REFUSED; return true; } - return txn_memtable_map_took_pushed(took, open, hold, key, key_len, reserve, answer, &value, has_was, was, error); + return txn_memtable_map_took_pushed(took, open, hold, found, key, key_len, reserve, answer, &value, has_was, was, + error); } /* * txn_memtable_map_took_core is took's body at every kept value: inside the * window, under the take's range's lock, a take with a claim is refused when - * the claim was overtaken; the step reads the key's newest entry in the - * delta, else the base's value given, and answers a put, a removal, a try + * the claim was overtaken, and one with a read is answered moved when the + * key's newest entry in the delta, else the base's value given, or the base + * generation is no longer what its caller read (bugs row 342); the step + * reads that entry, and answers a put, a removal, a try * again, or a refusal; a put or a removal is pushed with the reserve, its * tuple id the one the kept value names, a free space map's put lowers the * state's hints to the block, and then runs on the kept value. step, tuple, * and then share captures, through which the kept value goes; each runs at - * most once. + * most once. A pushed version counts at its writer's entry when it names + * one. */ bool txn_memtable_map_took_core(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t take, - TxnMemtableVersionWriter writer, bool has_lsn, uint64 lsn, bool has_claim, + TxnMemtableVersionWriter writer, bool has_roomed, TxnStatusRoomed roomed, bool has_lsn, + uint64 lsn, bool has_claim, HostRow claim_tid, uint64 claim_bytes, bool has_base, const uint8 *base, size_t base_len, + const TxnMemtableMapRead *read, TxnMemtableMapStep (*step) (void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out), HostRow (*tuple) (void *captures), void (*then) (void *captures), void *captures, @@ -1334,13 +1715,14 @@ txn_memtable_map_took_core(HostIoTakesShort io, TxnMemtableTablePreparedState *p const TxnMemtableTableTreeOps *tree; txn_memtable_map_took_t took = {.writer = writer, .has_lsn = has_lsn, .lsn = lsn, .has_claim = has_claim, .claim_tid = claim_tid, .claim_bytes = claim_bytes, .has_base = has_base, .base = base, .base_len = base_len, - .step = step, .tuple = tuple, .then = then, .captures = captures, .result = result}; + .read = read, .step = step, .tuple = tuple, .then = then, .captures = captures, .result = result}; if (!txn_memtable_table_Prepared_aimed(prepared, target, &state, &tree, error)) return false; took.state = state; took.free = tree == &TXN_MEMTABLE_MAP_FREE_TREE_OPS; - return txn_memtable_table_within(io, prepared, target, txn_memtable_map_took_step, &took, error); + return txn_memtable_table_within(io, prepared, target, has_roomed, roomed, txn_memtable_map_took_step, &took, + error); } /* The map's takes that keep nothing, and its prepare whose closure returns nothing, which map.c defines. */ @@ -1503,8 +1885,9 @@ txn_memtable_map_tests_taken_inside(void *captures, HostIoIo *io, TxnMemtableTab TxnStatusRoomed roomed = {0}; c->ran = 0; - if (!txn_memtable_map_finish_Seen(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, txn_memtable_testing_writer(7), false, 0, - true, (const uint8 *) "base", 4, txn_memtable_map_tests_taken_step, c, NULL, + if (!txn_memtable_map_finish_Seen(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, txn_memtable_testing_writer(7), false, + roomed, false, 0, + true, (const uint8 *) "base", 4, NULL, txn_memtable_map_tests_taken_step, c, NULL, txn_memtable_map_tests_taken_then, c, NULL, result_0, result_1, error)) return false; if (!txn_memtable_table_finish(HOST_IO_IO_TAKES_SHORT(io), prepared, c->change, false, roomed, @@ -1631,6 +2014,8 @@ txn_memtable_map_tests_taken_Debug_fmt(const void *value, StdFormatter *f) return std_DebugTuple_finish(&tuple); case TXN_MEMTABLE_MAP_TAKEN_RETRY: return std_fmt_write_str(f, "Retry", 5); + case TXN_MEMTABLE_MAP_TAKEN_MOVED: + return std_fmt_write_str(f, "Moved", 5); default: return std_fmt_write_str(f, "Refused", 7); } @@ -2152,8 +2537,9 @@ txn_memtable_map_tests_map_only_inside(void *captures, HostIoIo *io, TxnMemtable TxnStatusRoomed roomed = {0}; HostError refused; - if (!txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, txn_memtable_testing_writer(7), false, 0, - false, NULL, 0, txn_memtable_map_tests_left, NULL, NULL, + if (!txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, txn_memtable_testing_writer(7), false, + roomed, false, 0, + false, NULL, 0, NULL, txn_memtable_map_tests_left, NULL, NULL, txn_memtable_map_tests_nothing, NULL, NULL, result_0, error)) return false; *result_1 = txn_memtable_table_finish(HOST_IO_IO_TAKES_SHORT(io), prepared, change, false, roomed, &handle, @@ -2190,6 +2576,8 @@ txn_memtable_map_tests_taken_unit_Debug_fmt(const void *value, StdFormatter *f) return std_DebugTuple_finish(&tuple); case TXN_MEMTABLE_MAP_TAKEN_RETRY: return std_fmt_write_str(f, "Retry", 5); + case TXN_MEMTABLE_MAP_TAKEN_MOVED: + return std_fmt_write_str(f, "Moved", 5); default: return std_fmt_write_str(f, "Refused", 7); } @@ -2276,6 +2664,81 @@ test_txn_memtable_map_tests_a_map_only_prepare_takes_its_entry_and_makes_no_rows simc_TestIo_drop(&test_io); } +/* txn_memtable_map_tests_read_t is the moved case's take: the read it compares and the value it puts. */ +typedef struct txn_memtable_map_tests_read_t +{ + const TxnMemtableMapRead *read; + const char *value; +} txn_memtable_map_tests_read_t; + +/* txn_memtable_map_tests_put_value is the moved case's step: it puts its value. */ +static TxnMemtableMapStep +txn_memtable_map_tests_put_value(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out) +{ + const char *value = captures; + + std_vec_extend_from_slice_Checked(out, (const uint8 *) value, strlen(value), 1, 1); + return TXN_MEMTABLE_MAP_STEP_PUT; +} + +/* txn_memtable_map_tests_read_inside is the moved case's prepare's closure: the take's finish with its read. */ +static bool +txn_memtable_map_tests_read_inside(void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, + TxnMemtableMapTaken *result, HostError *error) +{ + const txn_memtable_map_tests_read_t *c = captures; + TxnStatusRoomed roomed = {0}; + + return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, txn_memtable_testing_writer(7), false, + roomed, false, 0, false, NULL, 0, c->read, txn_memtable_map_tests_put_value, + (void *) c->value, NULL, txn_memtable_map_tests_nothing, NULL, NULL, result, + error); +} + +/* + * The moved case (bugs row 342): a take whose caller read an entry its + * finish no longer finds answers moved and pushes nothing, and one whose + * read the delta still holds takes. + */ +void +test_txn_memtable_map_tests_a_take_whose_read_moved_answers_moved_and_pushes_nothing(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9362); + HostSharedRecord state = host_shared_Held_record(&held); + TxnMemtableMapTake take = {.state = state, .map = TXN_MEMTABLE_MAP_MAP_FREE, .key = 7, .most = 8}; + TxnMemtableMapRead none = {.generation = txn_state_generation(state)}; + TxnMemtableMapRead stale = {.has_entry = true, .entry = (const uint8 *) "stale", .entry_len = 5, + .generation = txn_state_generation(state)}; + TxnMemtableMapRead read = {.has_entry = true, .entry = (const uint8 *) "old", .entry_len = 3, + .generation = txn_state_generation(state)}; + txn_memtable_map_tests_read_t first = {&none, "old"}; + txn_memtable_map_tests_read_t moved = {&stale, "new"}; + txn_memtable_map_tests_read_t kept = {&read, "new"}; + TxnMemtableMapEntry old = txn_memtable_map_tests_entry("old"); + TxnMemtableMapEntry new = txn_memtable_map_tests_entry("new"); + TxnMemtableMapTaken taken; + HostError error; + + if (!txn_memtable_map_prepare_Taken(io, &take, 1, txn_memtable_map_tests_read_inside, &first, NULL, &taken, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, taken == TXN_MEMTABLE_MAP_TAKEN_TOOK, "a take whose read the delta holds takes"); + if (!txn_memtable_map_prepare_Taken(io, &take, 1, txn_memtable_map_tests_read_inside, &moved, NULL, &taken, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, taken == TXN_MEMTABLE_MAP_TAKEN_MOVED, "a take whose read moved answers moved"); + txn_memtable_map_tests_newest_is(__LINE__, 0, __LINE__, 0, io, state, TXN_MEMTABLE_MAP_MAP_FREE, 7, true, &old); + if (!txn_memtable_map_prepare_Taken(io, &take, 1, txn_memtable_map_tests_read_inside, &kept, NULL, &taken, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, taken == TXN_MEMTABLE_MAP_TAKEN_TOOK, "a take whose read is the delta's newest takes"); + txn_memtable_map_tests_newest_is(__LINE__, 0, __LINE__, 0, io, state, TXN_MEMTABLE_MAP_MAP_FREE, 7, true, &new); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + #define TXN_MEMTABLE_TABLE_RS Unit #define TXN_MEMTABLE_TABLE_RETURNS bool #define TXN_MEMTABLE_TABLE_RESULT_PARAMS , HostError *error @@ -2306,7 +2769,8 @@ txn_memtable_map_tests_window_inside(void *captures, HostIoIo *io, TxnMemtableTa for (size_t take = 0; take < lengthof(values); take++) if (!txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, take, txn_memtable_testing_writer(7), - false, 0, false, NULL, 0, txn_memtable_map_tests_written, + false, roomed, + false, 0, false, NULL, 0, NULL, txn_memtable_map_tests_written, (void *) values[take], NULL, txn_memtable_map_tests_nothing, NULL, NULL, &taken, error)) return false; diff --git a/src/doltgres/txn/c/memtable/pages.c b/src/doltgres/txn/c/memtable/pages.c index 3475fff1ba6e0..9796ff5eb7962 100644 --- a/src/doltgres/txn/c/memtable/pages.c +++ b/src/doltgres/txn/c/memtable/pages.c @@ -38,9 +38,6 @@ #define RUST_FILE "txn" STD_SEP "src" STD_SEP "memtable" STD_SEP "pages.rs" -/* TXN_MEMTABLE_PAGES_SMALLEST is SMALLEST, the body of a range's first page, before what it must hold makes it larger. */ -#define TXN_MEMTABLE_PAGES_SMALLEST (UINT64CONST(1) << 10) - /* TXN_MEMTABLE_PAGES_LARGER is LARGER, the body a range's pages double to, past which a page holds one record of its own. */ #define TXN_MEMTABLE_PAGES_LARGER (UINT64CONST(64) << 10) diff --git a/src/doltgres/txn/c/memtable/slice.c b/src/doltgres/txn/c/memtable/slice.c index 986c58e12cc69..60167a18eac49 100644 --- a/src/doltgres/txn/c/memtable/slice.c +++ b/src/doltgres/txn/c/memtable/slice.c @@ -1297,7 +1297,7 @@ txn_memtable_slice_row_versions_body_0(void *frame_) } std_vec_new(&frame->spilled, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); if (!txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->hold, frame->state, TXN_RUNS_KIND_SPILLED_ROWS, - frame->key, frame->key_len, &frame->spilled, frame->error)) + frame->key, frame->key_len, NULL, &frame->spilled, frame->error)) { txn_memtable_slice_row_versions_pad_0(frame); return; diff --git a/src/doltgres/txn/c/memtable/table.c b/src/doltgres/txn/c/memtable/table.c index 3dacf4b2a50b8..e39e23009b856 100644 --- a/src/doltgres/txn/c/memtable/table.c +++ b/src/doltgres/txn/c/memtable/table.c @@ -28,6 +28,7 @@ #endif #include "doltgres/host/delta.h" +#include "doltgres/host/delta/blocks.h" #include "doltgres/host/delta/list.h" #include "doltgres/host/delta/page.h" #include "doltgres/host/shared.h" @@ -335,18 +336,38 @@ txn_memtable_table_Reserve_drop(TxnMemtableTableReserve *self) /* * txn_memtable_table_Reserve_push makes a version its key's newest in an * open range with the reserve, which always has room, and keeps what the - * push saw of the range for the settle. + * push saw of the range for the settle. A version but a lock's that the + * reserve's step counts carries the counted mark. */ bool txn_memtable_table_Reserve_push(TxnMemtableTableReserve *self, TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, const uint8 *key, size_t key_len, const TxnMemtableVersionChange *change, TxnMemtableVersionHandle *result, HostError *error) +{ + TxnMemtableIndexFound found; + + return txn_memtable_index_lookup(open, hold, key, key_len, &found, error) && + txn_memtable_table_Reserve_push_found(self, open, hold, found, key, key_len, change, result, error); +} + +/* + * txn_memtable_table_Reserve_push_found is Reserve_push with the key's lookup + * in the same open already made, its Found taken by value as + * txn_memtable_version_push_found takes it. + */ +bool +txn_memtable_table_Reserve_push_found(TxnMemtableTableReserve *self, TxnMemtableIndexKeyedOpen *open, + const HostDeltaHold *hold, TxnMemtableIndexFound found, const uint8 *key, + size_t key_len, const TxnMemtableVersionChange *change, + TxnMemtableVersionHandle *result, HostError *error) { TxnMemtableVersionPushed pushed; HostSharedRecord record = host_shared_Guard_record(open->open.guard); HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; HostDeltaListListView pages; + bool counted = self->counting && change->op->tag != TXN_MEMTABLE_OP_LOCK && + change->op->tag != TXN_MEMTABLE_OP_LOCK_ROW; if (txn_memtable_version_encoded(&change->writer, change->op, change->shown, change->has_lsn, change->lsn) > self->buffer.cap) @@ -354,8 +375,8 @@ txn_memtable_table_Reserve_push(TxnMemtableTableReserve *self, TxnMemtableIndexK *error = txn_memtable_table_internal("a prepared version is longer than its prepare's record"); return false; } - if (!txn_memtable_version_push(open, hold, key, key_len, change, - &self->has_fresh, &self->fresh, &self->spare, &self->buffer, &pushed, error)) + if (!txn_memtable_version_push_found(open, hold, found, key, key_len, change, counted, + &self->has_fresh, &self->fresh, &self->spare, &self->buffer, &pushed, error)) return false; if (pushed.tag != TXN_MEMTABLE_VERSION_PUSHED_VERSION) { @@ -369,6 +390,7 @@ txn_memtable_table_Reserve_push(TxnMemtableTableReserve *self, TxnMemtableIndexK self->seen.bytes = host_shared_Record_atomic(record, TXN_MEMTABLE_RANGE_RANGE_BYTES, load); self->seen.pages = host_delta_list_ListView_len(&pages); self->seen.longest = (uint64) key_len; + self->counted = self->counted || counted; *result = pushed.u.version; return true; } @@ -502,25 +524,21 @@ txn_memtable_table_passed(HostIoTakesShort io, const HostDeltaHold *hold, uint64 } /* - * txn_memtable_table_reached runs a step under the exclusive lock of the - * open range that now holds a key, from the range a hand-off found: a closed - * range's replacement words name its halves, stored before closed and read - * with no lock, of which the key's is picked by the upper's first key, and a - * closed range with none sends it to a search of the memtable's list; each - * step takes one lock at a time, opens only what the hold reaches, and - * allocates nothing. + * txn_memtable_table_opened takes the exclusive lock of the open range that + * now holds a key, from the range a hand-off found, answering its guard and + * its open: a closed range's replacement words name its halves, stored + * before closed and read with no lock, of which the key's is picked by the + * upper's first key, and a closed range with none sends it to a search of + * the memtable's list; each step takes one lock at a time, opens only what + * the hold reaches, and allocates nothing. */ static bool -txn_memtable_table_reached(HostIoTakesShort io, const HostDeltaHold *hold, TxnMemtableTableAt *at, const uint8 *key, - size_t key_len, bool (*step) (void *captures, TxnMemtableIndexKeyedOpen *open, - HostError *error), - void *captures, HostError *error) +txn_memtable_table_opened(HostIoTakesShort io, const HostDeltaHold *hold, TxnMemtableTableAt *at, const uint8 *key, + size_t key_len, HostSharedGuard *guard, TxnMemtableIndexKeyedOpen *open, HostError *error) { for (;;) { HostSharedRecord record; - HostSharedGuard guard; - TxnMemtableIndexKeyedOpen open; HostSharedPast lower; HostSharedPast upper; uint64 lower_at; @@ -529,20 +547,15 @@ txn_memtable_table_reached(HostIoTakesShort io, const HostDeltaHold *hold, TxnMe bool has_upper; if (!host_delta_Hold_record(hold, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS, at->range, &record, error) || - !host_shared_Record_lock(record, io, TXN_MEMTABLE_RANGE_RANGE_LOCK, &guard, error)) + !host_shared_Record_lock(record, io, TXN_MEMTABLE_RANGE_RANGE_LOCK, guard, error)) return false; - if (txn_memtable_index_keyed_open(&TXN_MEMTABLE_RANGE_RANGE_KEYED_OPS, &guard, &open)) - { - bool ok = step(captures, &open, error); - - host_shared_Guard_drop(&guard); - return ok; - } + if (txn_memtable_index_keyed_open(&TXN_MEMTABLE_RANGE_RANGE_KEYED_OPS, guard, open)) + return true; lower = host_shared_Record_names_past(record, TXN_MEMTABLE_RANGE_RANGE_LOWER); upper = host_shared_Record_names_past(record, TXN_MEMTABLE_RANGE_RANGE_UPPER); has_lower = host_shared_Past_load(&lower, &lower_at); has_upper = host_shared_Past_load(&upper, &upper_at); - host_shared_Guard_drop(&guard); + host_shared_Guard_drop(guard); if (has_lower && has_upper) { HostDeltaLent first; @@ -560,17 +573,72 @@ txn_memtable_table_reached(HostIoTakesShort io, const HostDeltaHold *hold, TxnMe } } +/* txn_memtable_table_reached runs a step under the exclusive lock of the open range that now holds a key (opened). */ +static bool +txn_memtable_table_reached(HostIoTakesShort io, const HostDeltaHold *hold, TxnMemtableTableAt *at, const uint8 *key, + size_t key_len, bool (*step) (void *captures, TxnMemtableIndexKeyedOpen *open, + HostError *error), + void *captures, HostError *error) +{ + HostSharedGuard guard; + TxnMemtableIndexKeyedOpen open; + bool ok; + + if (!txn_memtable_table_opened(io, hold, at, key, key_len, &guard, &open, error)) + return false; + ok = step(captures, &open, error); + host_shared_Guard_drop(&guard); + return ok; +} + +static bool txn_memtable_table_deferred(HostIoIo *io, bool *result, HostError *error); + +/* + * A memtable's first range's page, which holds the first node alone, is the + * smallest range page, a quarter of its body words, and that body is one + * slot of the smallest class (bugs row 338). + */ +StaticAssertDecl(TXN_MEMTABLE_PAGES_FIRST + TXN_MEMTABLE_INDEX_LEAST_WORDS <= TXN_MEMTABLE_PAGES_SMALLEST / 32 && + TXN_MEMTABLE_PAGES_SMALLEST <= HOST_DELTA_PAGE_SLOT_BYTES, + "a memtable's first page is one slot of the smallest class"); + +/* txn_memtable_table_first_size returns the size of a memtable's first range's page. */ +static HostDeltaPageSize +txn_memtable_table_first_size(void) +{ + return txn_memtable_pages_size(false, 0, 0, txn_memtable_index_least(NULL, 0).words); +} + /* txn_memtable_table_prepared_frame_0_t is prepared's state across its pad: the sizes, the hold, the hand-offs, the pages, the targets, and inside's closure, while live. */ typedef struct txn_memtable_table_prepared_frame_0_t { HostIoIo *io; + /* all and all_len are the caller's aims, which groups take in turn. */ + const TxnMemtableTableAim *all; + size_t all_len; const TxnMemtableTableAim *aims; size_t aims_len; + /* pinned is the count of aims from aims whose trees the try pinned. */ + size_t pinned; + /* cap is the most aims a group's next try takes, halved by a refusal. */ + size_t cap; bool (*inside) (void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, HostError *error); void *inside_captures; void (*inside_drop) (void *captures); bool *has_unstored; TxnMemtableTableReservation *unstored; + + /* + * refused, for a carry's prepare, answers whether the delta's budget + * refused it, with its refusal. + */ + TxnMemtableTableRefusal *refused; + + /* + * answered is whether a group answered the delta's refusal through + * refused. + */ + bool answered; HostError *error; bool ok; bool inside_live; @@ -587,16 +655,117 @@ typedef struct txn_memtable_table_prepared_frame_0_t bool targets_live; TxnMemtableTableTarget target; bool target_live; + + /* + * missing is a Vec of the aims whose trees a try makes a memtable + * for. + */ + StdVec missing; + bool missing_live; + + /* + * firsts is a Vec: the records and first page of each + * memtable a try makes, missing's last first, until it is linked. + */ + StdVec firsts; + bool firsts_live; } txn_memtable_table_prepared_frame_0_t; -/* txn_memtable_table_prepared_unpinned clears the process's pin of each aim's tree. */ +static bool txn_memtable_table_prepared_firsts(txn_memtable_table_prepared_frame_0_t *frame, bool *refused, + HostDeltaFull *full); +static bool txn_memtable_table_prepared_linked(txn_memtable_table_prepared_frame_0_t *frame, bool *some); +static void txn_memtable_table_prepared_firsts_drop(txn_memtable_table_prepared_frame_0_t *frame); + +/* txn_memtable_table_prepared_unpinned clears the process's pin of each pinned aim's tree. */ static void txn_memtable_table_prepared_unpinned(txn_memtable_table_prepared_frame_0_t *frame) { - for (size_t k = 0; k < frame->aims_len; k++) + for (size_t k = 0; k < frame->pinned; k++) txn_memtable_pins_unpin(&frame->mine, frame->aims[k].tree->pin); } +/* + * An aim's pages, its version page, a list page for each level, and its + * missing memtable's first page, each at most a block, fit the smallest + * delta_memory's blocks, 1MB's, so a group always takes its first aim (bugs + * row 338). + */ +StaticAssertDecl((2 + HOST_DELTA_LIST_DEEPEST) * HOST_DELTA_PAGE_BLOCK <= 1024 * 1024, + "an aim's pages fit the smallest delta_memory"); + +/* + * TXN_MEMTABLE_TABLE_WHOLE aims are one group at every delta_memory: each + * aim's version page takes at most a block, the list pages, a list page's + * words to a slot, pack into whole blocks, and so do the missing memtables' + * first pages, a slot of the smallest class each. + */ +StaticAssertDecl(TXN_MEMTABLE_TABLE_WHOLE + + (TXN_MEMTABLE_TABLE_WHOLE * HOST_DELTA_LIST_DEEPEST * HOST_DELTA_LIST_FANOUT * 8 + + HOST_DELTA_PAGE_BLOCK - 1) / HOST_DELTA_PAGE_BLOCK + + (TXN_MEMTABLE_TABLE_WHOLE * HOST_DELTA_PAGE_SLOT_BYTES + HOST_DELTA_PAGE_BLOCK - 1) / + HOST_DELTA_PAGE_BLOCK <= 1024 * 1024 / HOST_DELTA_PAGE_BLOCK, + "a prepare of WHOLE aims is one group at the smallest delta_memory"); + +/* + * txn_memtable_table_prepared_least returns the fewest aims a group takes: + * all of a prepare of at most TXN_MEMTABLE_TABLE_WHOLE aims, which never + * splits, and one of a larger one, whose inside takes groups. + */ +static size_t +txn_memtable_table_prepared_least(const txn_memtable_table_prepared_frame_0_t *frame) +{ + return frame->all_len <= TXN_MEMTABLE_TABLE_WHOLE ? frame->aims_len : 1; +} + +/* + * txn_memtable_table_prepared_fitting returns how many of the try's aims, + * from its first, its group takes: its least, and then, to its cap, each one + * more whose pages, a missing memtable's first page with its aim's, their + * slots counted by class into whole blocks with the + * group's, fit the budget's blocks no page takes now, so no allocation of a + * prepare asks more than an empty budget holds, and a large one asks what the + * budget has free (bugs row 338). + */ +static size_t +txn_memtable_table_prepared_fitting(const txn_memtable_table_prepared_frame_0_t *frame, size_t each, + HostDeltaPageSize first) +{ + size_t missing = 0; + uint64 budgeted = host_delta_budget() / HOST_DELTA_PAGE_BLOCK; + uint64 used = host_delta_used() / HOST_DELTA_PAGE_BLOCK; + uint64 free = budgeted > used ? budgeted - used : 0; + size_t least = txn_memtable_table_prepared_least(frame); + uint64 taken[HOST_DELTA_BLOCKS_CLASSES] = {0}; + + for (size_t k = 0; k < frame->aims_len; k++) + { + uint64 with[HOST_DELTA_BLOCKS_CLASSES]; + uint64 blocks = 0; + + memcpy(with, taken, sizeof(taken)); + for (size_t d = 0; d < each; d++) + { + HostDeltaPageSize size = STD_VEC_AT(&frame->sizes, HostDeltaPageSize, k * each + d); + uint32 class = host_delta_blocks_class(host_delta_page_Size_body(size)); + + with[class] += HOST_DELTA_PAGE_SLOT_BYTES << class; + } + if (missing < frame->missing.len && STD_VEC_AT(&frame->missing, size_t, missing) == k) + { + uint32 class = host_delta_blocks_class(host_delta_page_Size_body(first)); + + with[class] += HOST_DELTA_PAGE_SLOT_BYTES << class; + missing++; + } + for (uint32 c = 0; c < HOST_DELTA_BLOCKS_CLASSES; c++) + blocks += (with[c] + HOST_DELTA_PAGE_BLOCK - 1) / HOST_DELTA_PAGE_BLOCK; + if (k >= least && (blocks > free || k >= frame->cap)) + return k; + memcpy(taken, with, sizeof(taken)); + } + return frame->aims_len; +} + /* txn_memtable_table_prepared_pages_free drops the pages a try allocated that no target took. */ static void txn_memtable_table_prepared_pages_free(txn_memtable_table_prepared_frame_0_t *frame) @@ -610,7 +779,7 @@ txn_memtable_table_prepared_pages_free(txn_memtable_table_prepared_frame_0_t *fr } } -/* txn_memtable_table_prepared_pad_0 is prepared's pad: the targets, the pages, the hold, the hand-offs, the sizes, and inside's closure, while live. */ +/* txn_memtable_table_prepared_pad_0 is prepared's pad: the targets, the memtables unlinked, the pages, the hold, the hand-offs, the sizes, and inside's closure, while live. */ static void txn_memtable_table_prepared_pad_0(void *frame_) { @@ -628,6 +797,7 @@ txn_memtable_table_prepared_pad_0(void *frame_) std_vec_free_Checked(&frame->prepared.targets, sizeof(TxnMemtableTableTarget), _Alignof(TxnMemtableTableTarget)); } + txn_memtable_table_prepared_firsts_drop(frame); txn_memtable_table_prepared_pages_free(frame); if (frame->hold_live) { @@ -644,6 +814,11 @@ txn_memtable_table_prepared_pad_0(void *frame_) frame->sizes_live = false; std_vec_free_Checked(&frame->sizes, sizeof(HostDeltaPageSize), _Alignof(HostDeltaPageSize)); } + if (frame->missing_live) + { + frame->missing_live = false; + std_vec_free_Checked(&frame->missing, sizeof(size_t), _Alignof(size_t)); + } if (frame->inside_live) { frame->inside_live = false; @@ -653,22 +828,38 @@ txn_memtable_table_prepared_pad_0(void *frame_) } /* - * txn_memtable_table_prepared_try is one try of prepared's: it makes each - * memtable a tree lacks, begins the hold, hands each push off, and takes - * every push's reserve, answering false on an error and setting ready once - * the frame holds the targets. + * txn_memtable_table_prepared_try is one try of prepared's: it begins the + * hold, which a readied prepare keeps; hands each push to a tree with a + * memtable off to its key's range, pinning its memtable; takes the reserves + * of the pushes that fit the budget, and the first page of each of their + * memtables a tree lacks, in one allocation; then takes the records of every one of those memtables before + * it makes any, and then makes each on its page as its first writer and + * hands its push off. A refused try takes nothing, the refusal of any one + * memtable's records included, so a refused writer leaves no memtable for a + * walk to close and free. It answers + * false on an error, sets ready once the frame holds the targets, and sets + * refused, with the delta's refusal, when the delta refused the pages of a + * group's least or a memtable's records, the try's pins given back. */ static bool -txn_memtable_table_prepared_try(txn_memtable_table_prepared_frame_0_t *frame, bool *ready) +txn_memtable_table_prepared_try(txn_memtable_table_prepared_frame_0_t *frame, bool *ready, bool *refused, + HostDeltaFull *full) { size_t each = 1 + HOST_DELTA_LIST_DEEPEST; + size_t sized = std_wrapping_mul_usize(frame->aims_len, each); + HostDeltaPageSize first = txn_memtable_table_first_size(); + HostDeltaPageSize list_page = HOST_DELTA_LIST_PAGE; bool allocated; - HostDeltaFull full; + size_t kept = 0; *ready = false; - for (size_t k = 0; k < frame->aims_len; k++) - if (!txn_memtable_table_first(frame->aims[k].tree, frame->io, frame->aims[k].owner, frame->error)) - return false; + *refused = false; + frame->pinned = frame->aims_len; + frame->sizes.len = sized; + if (frame->missing_live) + std_vec_free_Checked(&frame->missing, sizeof(size_t), _Alignof(size_t)); + std_vec_new(&frame->missing, sizeof(size_t), _Alignof(size_t)); + frame->missing_live = true; if (!host_delta_hold(frame->io, &frame->prepared.hold, frame->error)) return false; frame->hold_live = true; @@ -682,43 +873,91 @@ txn_memtable_table_prepared_try(txn_memtable_table_prepared_frame_0_t *frame, bo for (size_t k = 0; k < frame->aims_len; k++) { const TxnMemtableTableAim *aim = &frame->aims[k]; - bool some; - TxnMemtableTableAt at; + TxnMemtableTableAt at = {0}; + uint64 location; + uint64 newest = first.words; - if (!txn_memtable_table_handed(aim->tree, frame->io, &frame->prepared.hold, &frame->mine, aim->owner, - aim->key.ptr, aim->key.len, &some, &at, frame->error)) - return false; - if (!some) + if (!txn_memtable_table_memtable(aim->tree, aim->owner, &location)) { - txn_memtable_table_prepared_unpinned(frame); - return true; + if (!std_vec_try_reserve(&frame->missing, 1, sizeof(size_t), _Alignof(size_t)) || + !std_vec_try_reserve(&frame->sizes, 1, sizeof(HostDeltaPageSize), _Alignof(HostDeltaPageSize))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(&frame->missing, &k, sizeof(size_t), _Alignof(size_t)); + std_vec_push_Checked(&frame->sizes, &first, sizeof(first), _Alignof(HostDeltaPageSize)); } - std_vec_push_Checked(&frame->ats, &at, sizeof(at), _Alignof(TxnMemtableTableAt)); - } - for (size_t k = 0; k < frame->aims_len; k++) - { - HostSharedRecord range; - HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + else + { + HostSharedRecord range; + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + bool some; - if (!host_delta_Hold_record(&frame->prepared.hold, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS, - STD_VEC_AT(&frame->ats, TxnMemtableTableAt, k).range, &range, frame->error)) - return false; + if (!txn_memtable_table_handed(aim->tree, frame->io, &frame->prepared.hold, &frame->mine, aim->owner, + aim->key.ptr, aim->key.len, &some, &at, frame->error)) + return false; + if (!some) + { + txn_memtable_table_prepared_unpinned(frame); + return true; + } + if (!host_delta_Hold_record(&frame->prepared.hold, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS, at.range, &range, + frame->error)) + return false; + newest = host_shared_Record_atomic(range, TXN_MEMTABLE_RANGE_RANGE_NEWEST, load); + } + std_vec_push_Checked(&frame->ats, &at, sizeof(at), _Alignof(TxnMemtableTableAt)); STD_VEC_AT(&frame->sizes, HostDeltaPageSize, std_wrapping_mul_usize(k, each)) = - txn_memtable_version_reserve(host_shared_Record_atomic(range, TXN_MEMTABLE_RANGE_RANGE_NEWEST, load), - frame->aims[k].key.len, frame->aims[k].need); - } - if (!host_delta_alloc(frame->io, frame->sizes.ptr, frame->sizes.len, &allocated, &frame->pages, &full, + txn_memtable_version_reserve(newest, aim->key.len, aim->need); + for (size_t d = 1; d < each; d++) + STD_VEC_AT(&frame->sizes, HostDeltaPageSize, std_wrapping_mul_usize(k, each) + d) = list_page; + } + frame->aims_len = txn_memtable_table_prepared_fitting(frame, each, first); + for (size_t i = 0; i < frame->missing.len; i++) + if (STD_VEC_AT(&frame->missing, size_t, i) < frame->aims_len) + STD_VEC_AT(&frame->missing, size_t, kept++) = STD_VEC_AT(&frame->missing, size_t, i); + frame->missing.len = kept; + for (size_t i = 0; i < kept; i++) + STD_VEC_AT(&frame->sizes, HostDeltaPageSize, std_wrapping_mul_usize(frame->aims_len, each) + i) = first; + frame->sizes.len = std_wrapping_mul_usize(frame->aims_len, each) + kept; + frame->ats.len = frame->aims_len; + if (!host_delta_alloc(frame->io, frame->sizes.ptr, frame->sizes.len, &allocated, &frame->pages, full, frame->error)) return false; if (!allocated) { + size_t least = txn_memtable_table_prepared_least(frame); + txn_memtable_table_prepared_unpinned(frame); - frame->hold_live = false; - host_delta_Hold_drop(&frame->prepared.hold); - return txn_memtable_table_waited(frame->io, full, false, frame->error); + if (frame->aims_len > least) + frame->cap = Max(frame->aims_len / 2, least); + else + *refused = true; + return true; } frame->pages_live = true; frame->pages_at = 0; + if (!txn_memtable_table_prepared_firsts(frame, refused, full)) + return false; + if (*refused) + { + txn_memtable_table_prepared_unpinned(frame); + return true; + } + { + bool some; + + if (!txn_memtable_table_prepared_linked(frame, &some)) + return false; + if (!some) + { + txn_memtable_table_prepared_unpinned(frame); + return true; + } + } + frame->missing_live = false; + std_vec_free_Checked(&frame->missing, sizeof(size_t), _Alignof(size_t)); std_vec_new(&frame->prepared.targets, sizeof(TxnMemtableTableTarget), _Alignof(TxnMemtableTableTarget)); frame->targets_live = true; if (!std_vec_try_reserve(&frame->prepared.targets, frame->aims_len, sizeof(TxnMemtableTableTarget), @@ -768,48 +1007,39 @@ txn_memtable_table_prepared_try(txn_memtable_table_prepared_frame_0_t *frame, bo return true; } -/* txn_memtable_table_prepared_body_0 is prepared's body. */ -static void -txn_memtable_table_prepared_body_0(void *frame_) +/* + * txn_memtable_table_prepared_group readies the aims from start that fit an + * empty budget, as many as its try keeps, runs inside with them, and settles + * them, answering false on an error, or ends at a refusal it answered. + */ +static bool +txn_memtable_table_prepared_group(txn_memtable_table_prepared_frame_0_t *frame, size_t start) { - txn_memtable_table_prepared_frame_0_t *frame = frame_; size_t each = 1 + HOST_DELTA_LIST_DEEPEST; bool ready = false; bool ok; - HostDeltaPageSize list_page = HOST_DELTA_LIST_PAGE; + bool refused; + HostDeltaFull full; - if (!host_shared_mine(HOST_IO_IO_TAKES_SHORT(frame->io), &TXN_MEMTABLE_PINS_PINS_PROC_SLOTS_OPS, &frame->mine, - frame->error)) - { - txn_memtable_table_prepared_pad_0(frame); - return; - } - std_vec_new(&frame->sizes, sizeof(HostDeltaPageSize), _Alignof(HostDeltaPageSize)); - frame->sizes_live = true; + frame->aims = frame->all + start; + frame->aims_len = frame->all_len - start; + frame->cap = frame->aims_len; + frame->sizes.len = 0; if (!std_vec_try_reserve(&frame->sizes, STD_MUL(usize, frame->aims_len, each, 454, 23), sizeof(HostDeltaPageSize), _Alignof(HostDeltaPageSize))) { *frame->error = host_Error_out_of_memory(); - txn_memtable_table_prepared_pad_0(frame); - return; - } - for (size_t k = 0; k < frame->aims_len; k++) - { - HostDeltaPageSize size = txn_memtable_version_reserve(0, frame->aims[k].key.len, frame->aims[k].need); - - std_vec_push_Checked(&frame->sizes, &size, sizeof(size), _Alignof(HostDeltaPageSize)); - for (size_t d = 0; d < HOST_DELTA_LIST_DEEPEST; d++) - std_vec_push_Checked(&frame->sizes, &list_page, sizeof(list_page), _Alignof(HostDeltaPageSize)); + return false; } while (!ready) { - if (!txn_memtable_table_prepared_try(frame, &ready)) - { - txn_memtable_table_prepared_pad_0(frame); - return; - } + frame->aims_len = frame->all_len - start; + if (!txn_memtable_table_prepared_try(frame, &ready, &refused, &full)) + return false; if (!ready) { + bool answered; + txn_memtable_table_prepared_pages_free(frame); if (frame->ats_live) { @@ -821,16 +1051,24 @@ txn_memtable_table_prepared_body_0(void *frame_) frame->hold_live = false; host_delta_Hold_drop(&frame->prepared.hold); } + if (refused && !txn_memtable_table_refused(frame->io, full, frame->refused, &answered, frame->error)) + return false; + if (refused && answered) + { + frame->answered = true; + return true; + } } } frame->prepared.aims = frame->aims; frame->prepared.aims_len = frame->aims_len; + frame->prepared.first = start; ok = host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-memtable-prepared", frame->error); - frame->inside_live = false; if (ok) + { + frame->inside_live = false; ok = frame->inside(frame->inside_captures, frame->io, &frame->prepared, frame->error); - else if (frame->inside_drop != NULL) - frame->inside_drop(frame->inside_captures); + } txn_memtable_table_prepared_unpinned(frame); frame->hold_live = false; host_delta_Hold_drop(&frame->prepared.hold); @@ -846,21 +1084,62 @@ txn_memtable_table_prepared_body_0(void *frame_) std_vec_clear(&reserve->spare, sizeof(HostDeltaOwned), host_delta_Owned_drop); } if (!ok || !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-memtable-unpinned", frame->error)) - { - txn_memtable_table_prepared_pad_0(frame); - return; - } + return false; for (size_t k = 0, k_len = frame->prepared.targets.len; k < k_len; k++) { const TxnMemtableTableTarget *target = &STD_VEC_AT(&frame->prepared.targets, TxnMemtableTableTarget, k); if (!txn_memtable_table_settled(target->tree, frame->io, frame->aims[k].owner, target->at, target->reserve.has_seen, &target->reserve.seen, frame->error)) + return false; + } + frame->targets_live = false; + std_vec_clear(&frame->prepared.targets, sizeof(TxnMemtableTableTarget), txn_memtable_table_Target_drop); + std_vec_free_Checked(&frame->prepared.targets, sizeof(TxnMemtableTableTarget), _Alignof(TxnMemtableTableTarget)); + return true; +} + +/* + * txn_memtable_table_prepared_body_0 is prepared's body: its groups in turn, + * each the aims after the last group that an empty budget can give (bugs row + * 338). + */ +static void +txn_memtable_table_prepared_body_0(void *frame_) +{ + txn_memtable_table_prepared_frame_0_t *frame = frame_; + size_t start = 0; + + if (!host_shared_mine(HOST_IO_IO_TAKES_SHORT(frame->io), &TXN_MEMTABLE_PINS_PINS_PROC_SLOTS_OPS, &frame->mine, + frame->error)) + { + txn_memtable_table_prepared_pad_0(frame); + return; + } + std_vec_new(&frame->sizes, sizeof(HostDeltaPageSize), _Alignof(HostDeltaPageSize)); + frame->sizes_live = true; + do + { + if (!txn_memtable_table_prepared_group(frame, start)) + { + if (frame->inside_live && frame->inside_drop != NULL) + { + frame->inside_live = false; + frame->inside_drop(frame->inside_captures); + } + txn_memtable_table_prepared_pad_0(frame); + return; + } + if (frame->answered) { + Assert(start == 0); + *frame->has_unstored = false; + frame->ok = true; txn_memtable_table_prepared_pad_0(frame); return; } - } + start += frame->aims_len; + } while (start < frame->all_len); *frame->has_unstored = frame->prepared.has_tuple; if (frame->prepared.has_tuple) *frame->unstored = frame->prepared.tuple; @@ -872,12 +1151,17 @@ txn_memtable_table_prepared_body_0(void *frame_) * txn_memtable_table_prepared readies pushes to its owners' trees before a * checkpoint window and runs the window's work with them, taking inside's * closure, whose captures carry its result and what the caller hands it. - * Each try makes the memtable a tree lacks as its first writer, with no - * hold; begins the hold, which it keeps; hands each push off to its key's - * range, pinning its memtable; and then takes every push's reserve in one - * allocation, going around when a hand-off or the allocation can't. It then - * runs inside, whose finishes allocate nothing, and settles: it clears its - * pins, gives back what the finishes left of the reserves, and, unless + * Each try hands each push off to its key's range, pinning its memtable, and + * takes in one allocation the reserves of the pushes from the group's first + * that an empty budget can give, with the records and first page of each + * memtable the group's trees lack, which it takes before it links any, going + * around when a hand-off or the allocation can't and meeting the delta + * budget's refusal as refused says. It then runs inside with that group, + * whose finishes allocate nothing, and settles, group after group, so no + * allocation asks more than the budget holds (bugs row 338); a prepare of at + * most TXN_MEMTABLE_TABLE_WHOLE pushes is one group, and a refused group of + * more than its least tries again with half as many. A group's settle clears + * its pins, gives back what the finishes left of the reserves, and, unless * inside failed, splits each range a finish left past SPLIT keys and closes * each memtable a prepare made or pinned that no finish reached. It answers * the tuple id table's reservation the prepare was given, if no finish @@ -889,16 +1173,46 @@ txn_memtable_table_prepared(HostIoIo *io, const TxnMemtableTableAim *aims, size_ bool (*inside) (void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, HostError *error), void *inside_captures, void (*inside_drop) (void *captures), bool *has_unstored, - TxnMemtableTableReservation *unstored, HostError *error) + TxnMemtableTableReservation *unstored, TxnMemtableTableRefusal *refused, HostError *error) { - txn_memtable_table_prepared_frame_0_t frame = {.io = io, .aims = aims, .aims_len = aims_len, .inside = inside, + txn_memtable_table_prepared_frame_0_t frame = {.io = io, .all = aims, .all_len = aims_len, .inside = inside, .inside_captures = inside_captures, .inside_drop = inside_drop, .has_unstored = has_unstored, - .unstored = unstored, .error = error, .inside_live = true, .prepared = {.has_tuple = has_tuple, .tuple = tuple}}; + .unstored = unstored, .refused = refused, .error = error, .inside_live = true, + .prepared = {.has_tuple = has_tuple, .tuple = tuple}}; + if (refused != NULL) + refused->refused = false; std_landing(&frame, txn_memtable_table_prepared_body_0, txn_memtable_table_prepared_pad_0); return frame.ok; } +/* + * txn_memtable_table_refused meets the delta budget's refusal of a prepare: + * a writer's, whose refused is NULL, waits for a free and tries again; a + * rewrite's carry's tries again after a deferred compaction or a spill freed + * the budget, and otherwise answers the refusal through refused, having + * taken nothing, so its caller, which holds the catalog, folds the new + * storage first. It answers whether it answered the refusal. + */ +bool +txn_memtable_table_refused(HostIoIo *io, HostDeltaFull full, TxnMemtableTableRefusal *refused, bool *result, + HostError *error) +{ + bool freed; + + *result = false; + if (refused == NULL) + return txn_memtable_table_waited(io, full, false, error); + if (!txn_memtable_table_deferred(io, &freed, error)) + return false; + if (!freed && !txn_spill_relieved(io, &freed, error)) + return false; + refused->refused = !freed; + refused->full = full; + *result = !freed; + return true; +} + /* txn_memtable_table_within_t is within's step's captures: the caller's step and its captures, the hold, the key, and the reserve. */ typedef struct txn_memtable_table_within_t { @@ -920,15 +1234,62 @@ txn_memtable_table_within_step(void *captures, TxnMemtableIndexKeyedOpen *open, within->reserve, error); } +/* + * txn_memtable_table_kept takes and keeps the exclusive lock of the open + * range that holds a prepared aim's key, before its caller's window, so no + * removal or other take moves the key's chain until unkept, as heap holds a + * buffer's lock across its critical section; every finish of the key runs + * on it meanwhile (bugs row 342). + */ +bool +txn_memtable_table_kept(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t index, HostError *error) +{ + TxnMemtableTableTarget *target = &STD_VEC_AT(&prepared->targets, TxnMemtableTableTarget, index); + + if (!txn_memtable_table_opened(io, &prepared->hold, &target->at, prepared->aims[index].key.ptr, + prepared->aims[index].key.len, &prepared->kept_guard, &prepared->kept_open, error)) + return false; + prepared->has_kept = true; + prepared->kept = index; + return true; +} + +/* txn_memtable_table_unkept releases the range kept keeps, while it keeps one. */ +void +txn_memtable_table_unkept(TxnMemtableTablePreparedState *prepared) +{ + if (!prepared->has_kept) + return; + prepared->has_kept = false; + host_shared_Guard_drop(&prepared->kept_guard); +} + +/* txn_memtable_table_keeps reports whether a prepared aim's key is the one whose range kept keeps. */ +static bool +txn_memtable_table_keeps(const TxnMemtableTablePreparedState *prepared, size_t index) +{ + const TxnMemtableTableAim *held = &prepared->aims[prepared->kept]; + const TxnMemtableTableAim *aim = &prepared->aims[index]; + + return prepared->has_kept && held->tree == aim->tree && held->owner.place == aim->owner.place && + std_bytes_Ord_cmp(held->key.ptr, held->key.len, aim->key.ptr, aim->key.len) == STD_ORDERING_EQUAL; +} + /* * txn_memtable_table_within runs a step of a finish inside the window on a * prepared target: it follows the target's range to the open range that * holds its key now and runs the step there, under that range's exclusive * lock, with the prepare's hold, the target's key, and its reserve, then - * marks a push the target made, which a key lock's view keeps. + * marks a push the target made, which a key lock's view keeps. In a tree + * that counts, a writer with a room counts its version: one is added at the + * room's entry before the step and taken back after it, a failed step's + * included, unless the step pushed a version it marked counted, so the count + * never falls below the versions that name the entry, and a step that pushed + * none leaves it as it was. */ bool txn_memtable_table_within(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t index, + bool has_roomed, TxnStatusRoomed roomed, bool (*step) (void *captures, TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, const uint8 *key, size_t key_len, TxnMemtableTableReserve *reserve, HostError *error), @@ -938,6 +1299,8 @@ txn_memtable_table_within(HostIoTakesShort io, TxnMemtableTablePreparedState *pr txn_memtable_table_within_t within; bool ok; uint64 grown; + bool counting; + HostError given; if (index >= prepared->aims_len || index >= prepared->targets.len) { @@ -949,8 +1312,24 @@ txn_memtable_table_within(HostIoTakesShort io, TxnMemtableTablePreparedState *pr { step, captures, &prepared->hold, &prepared->aims[index], &target->reserve }; - ok = txn_memtable_table_reached(io, &prepared->hold, &target->at, prepared->aims[index].key.ptr, - prepared->aims[index].key.len, txn_memtable_table_within_step, &within, error); + counting = has_roomed && prepared->aims[index].tree->counted; + target->reserve.counting = counting; + target->reserve.counted = false; + if (counting && !txn_status_counted(io, &prepared->hold, roomed, 1, error)) + return false; + if (txn_memtable_table_keeps(prepared, index)) + ok = txn_memtable_table_within_step(&within, &prepared->kept_open, error); + else + ok = txn_memtable_table_reached(io, &prepared->hold, &target->at, prepared->aims[index].key.ptr, + prepared->aims[index].key.len, txn_memtable_table_within_step, &within, + error); + if (counting && !target->reserve.counted) + { + if (ok) + ok = txn_status_counted(io, &prepared->hold, roomed, PG_UINT64_MAX, error); + else if (!txn_status_counted(io, &prepared->hold, roomed, PG_UINT64_MAX, &given)) + host_Error_drop(&given); + } if (target->reserve.has_seen) host_pushed(io); grown = target->reserve.grown; @@ -992,6 +1371,12 @@ typedef struct txn_memtable_table_readied_frame_0_t bool (*inside) (void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, HostError *error); void *inside_captures; void (*inside_drop) (void *captures); + + /* + * refused, for a carry's prepare, answers whether the delta's budget + * refused it, with its refusal. + */ + TxnMemtableTableRefusal *refused; HostError *error; bool ok; bool inside_live; @@ -1087,8 +1472,18 @@ txn_memtable_table_readied_body_0(void *frame_) if (frame->tuple && row_count > 0) { if (!txn_memtable_tuples_reserve(frame->io, frame->state, frame->rows[0].f0.ptr, frame->rows[0].f0.len, - &reservation, frame->error) || - !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-tuple-reserved", frame->error)) + &reservation, frame->refused, frame->error)) + { + txn_memtable_table_readied_pad_0(frame); + return; + } + if (frame->refused != NULL && frame->refused->refused) + { + frame->ok = true; + txn_memtable_table_readied_pad_0(frame); + return; + } + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-tuple-reserved", frame->error)) { txn_memtable_table_readied_pad_0(frame); return; @@ -1098,12 +1493,14 @@ txn_memtable_table_readied_body_0(void *frame_) frame->inside_live = false; if (!txn_memtable_table_prepared(frame->io, frame->aims.ptr, frame->aims.len, has_tuple, reservation, frame->inside, frame->inside_captures, frame->inside_drop, &has_unstored, - &unstored, frame->error)) + &unstored, frame->refused, frame->error)) { txn_memtable_table_readied_pad_0(frame); return; } txn_memtable_table_readied_pad_0(frame); + if (frame->refused != NULL && frame->refused->refused) + has_unstored = has_tuple; frame->ok = !has_unstored || row_count == 0 || txn_memtable_tuples_given(frame->io, frame->state, frame->error); } @@ -1112,18 +1509,22 @@ txn_memtable_table_readied_body_0(void *frame_) * prepare of map takes alone, and map takes, each in the storage its take * names, as prepare and map's prepare do, with the first row's tuple id slot * when tuple is set, taking inside's closure, whose captures carry its - * result and what the caller hands it. + * result and what the caller hands it, and meeting the delta budget's + * refusal as refused says (refused): a carry's refused prepare gives back + * the slot it reserved. */ bool txn_memtable_table_readied(HostIoIo *io, bool has_rows, HostSharedRecord state, const txn_TupleRefSliceU8Usize_t *rows, size_t rows_len, const TxnMemtableMapTake *takes, size_t takes_len, bool tuple, bool (*inside) (void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, HostError *error), - void *inside_captures, void (*inside_drop) (void *captures), HostError *error) + void *inside_captures, void (*inside_drop) (void *captures), + TxnMemtableTableRefusal *refused, HostError *error) { txn_memtable_table_readied_frame_0_t frame = {.io = io, .has_rows = has_rows, .state = state, .rows = rows, .rows_len = rows_len, .takes = takes, .takes_len = takes_len, .tuple = tuple, .inside = inside, - .inside_captures = inside_captures, .inside_drop = inside_drop, .error = error, .inside_live = true}; + .inside_captures = inside_captures, .inside_drop = inside_drop, .refused = refused, .error = error, + .inside_live = true}; std_landing(&frame, txn_memtable_table_readied_body_0, txn_memtable_table_readied_pad_0); return frame.ok; @@ -1149,22 +1550,20 @@ txn_memtable_table_pushed_step(void *captures, TxnMemtableIndexKeyedOpen *open, return txn_memtable_table_Reserve_push(reserve, open, hold, key, key_len, pushed->change, pushed->result, error); } -/* txn_memtable_table_pushed pushes a prepared target's version inside the window with its reserve, allocating nothing. */ +/* + * txn_memtable_table_pushed pushes a prepared target's version inside the + * window with its reserve, allocating nothing, and counts it at its writer's + * entry when it names one. + */ bool txn_memtable_table_pushed(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t index, - const TxnMemtableVersionChange *change, TxnMemtableVersionHandle *result, HostError *error) + const TxnMemtableVersionChange *change, bool has_roomed, TxnStatusRoomed roomed, + TxnMemtableVersionHandle *result, HostError *error) { txn_memtable_table_pushed_t pushed = {.change = change, .result = result}; - return txn_memtable_table_within(io, prepared, index, txn_memtable_table_pushed_step, &pushed, error); -} - -/* txn_memtable_table_counted_at counts a pushed version at its writer's entry when it names one. */ -static bool -txn_memtable_table_counted_at(HostIoTakesShort io, const TxnMemtableTablePreparedState *prepared, bool has_roomed, - TxnStatusRoomed roomed, HostError *error) -{ - return !has_roomed || txn_status_counted(io, &prepared->hold, roomed, error); + return txn_memtable_table_within(io, prepared, index, has_roomed, roomed, txn_memtable_table_pushed_step, &pushed, + error); } /* @@ -1207,8 +1606,8 @@ txn_memtable_table_finish_speculative(HostIoTakesShort io, TxnMemtableTablePrepa { txn_memtable_table_pushed_t pushed = {.change = change, .result = result, .token = token}; - return txn_memtable_table_within(io, prepared, 0, txn_memtable_table_speculated_step, &pushed, error) && - txn_memtable_table_counted_at(io, prepared, has_roomed, roomed, error); + return txn_memtable_table_within(io, prepared, 0, has_roomed, roomed, txn_memtable_table_speculated_step, &pushed, + error); } /* txn_memtable_table_rowed refuses a row index past the prepare's rows with an error of its text. */ @@ -1252,8 +1651,8 @@ txn_memtable_table_finish_showing(HostIoTakesShort io, TxnMemtableTablePreparedS txn_memtable_table_pushed_t pushed = {.change = change, .result = result, .ended = ended, .combo = combo}; return txn_memtable_table_rowed(prepared, row, "a row's finish names a row the prepare didn't ready", error) && - txn_memtable_table_within(io, prepared, row, txn_memtable_table_shown_step, &pushed, error) && - txn_memtable_table_counted_at(io, prepared, has_roomed, roomed, error); + txn_memtable_table_within(io, prepared, row, has_roomed, roomed, txn_memtable_table_shown_step, &pushed, + error); } /* @@ -1266,8 +1665,7 @@ txn_memtable_table_finished(HostIoTakesShort io, TxnMemtableTablePreparedState * const TxnMemtableVersionChange *change, bool has_roomed, TxnStatusRoomed roomed, TxnMemtableVersionHandle *result, HostError *error) { - return txn_memtable_table_pushed(io, prepared, index, change, result, error) && - txn_memtable_table_counted_at(io, prepared, has_roomed, roomed, error); + return txn_memtable_table_pushed(io, prepared, index, change, has_roomed, roomed, result, error); } /* @@ -1285,20 +1683,6 @@ txn_memtable_table_finish_row(HostIoTakesShort io, TxnMemtableTablePreparedState txn_memtable_table_finished(io, prepared, row, change, has_roomed, roomed, result, error); } -/* - * txn_memtable_table_counted counts a map version a transaction pushed - * inside its window at its writer's entry, as finish_row counts a row's, so - * the entry stays while the version does, and uncounted takes it back when - * the fold's drop removes it; a version written with no Roomed, by no - * transaction, counts nothing. - */ -bool -txn_memtable_table_counted(HostIoTakesShort io, const TxnMemtableTablePreparedState *prepared, bool has_roomed, - TxnStatusRoomed roomed, HostError *error) -{ - return txn_memtable_table_counted_at(io, prepared, has_roomed, roomed, error); -} - /* txn_memtable_table_newest_t is newest's step's captures: its answer. */ typedef struct txn_memtable_table_newest_t { @@ -1335,10 +1719,12 @@ txn_memtable_table_newest(HostIoTakesShort io, TxnMemtableTablePreparedState *pr TxnMemtableVersionHandle *result, HostError *error) { txn_memtable_table_newest_t newest = {some, result}; + TxnStatusRoomed unroomed = {0}; return txn_memtable_table_rowed(prepared, row, "a row's newest version names a row the prepare didn't ready", error) && - txn_memtable_table_within(io, prepared, row, txn_memtable_table_newest_step, &newest, error); + txn_memtable_table_within(io, prepared, row, false, unroomed, txn_memtable_table_newest_step, + &newest, error); } /* @@ -1501,13 +1887,16 @@ txn_memtable_table_holds(HostIoIo *io, HostSharedRecord state, const uint8 *key, { if (!txn_memtable_table_held(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, state, key, key_len, lsn, result, error)) return false; - return *result || txn_spill_holds(io, state, TXN_RUNS_KIND_SPILLED_ROWS, key, key_len, lsn, result, error); + return *result || txn_spill_holds(&TXN_SPILL_STATE_HOME_OPS, io, state, TXN_RUNS_KIND_SPILLED_ROWS, key, key_len, + lsn, result, error); } /* txn_memtable_table_alone_t is alone's inside's captures: the change and the handle it gives. */ typedef struct txn_memtable_table_alone_t { const TxnMemtableVersionChange *change; + bool has_roomed; + TxnStatusRoomed roomed; TxnMemtableVersionHandle *result; } txn_memtable_table_alone_t; @@ -1518,28 +1907,30 @@ txn_memtable_table_alone_inside(void *captures, HostIoIo *io, TxnMemtableTablePr { txn_memtable_table_alone_t *alone = captures; - return txn_memtable_table_pushed(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, alone->change, alone->result, error); + return txn_memtable_table_pushed(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, alone->change, alone->has_roomed, + alone->roomed, alone->result, error); } /* * txn_memtable_table_alone makes a version a key's newest in one of an * owner's trees with no window, a prepare of that one push, its finish, and - * its settle in one call, as a row's push and the runs' loader do. + * its settle in one call, as a row's push and the runs' loader do, counting + * it at its writer's entry when it names one. */ bool txn_memtable_table_alone(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, const uint8 *key, - size_t key_len, const TxnMemtableVersionChange *change, TxnMemtableVersionHandle *result, - HostError *error) + size_t key_len, const TxnMemtableVersionChange *change, bool has_roomed, + TxnStatusRoomed roomed, TxnMemtableVersionHandle *result, HostError *error) { TxnMemtableTableAim aims[1] = {{.owner = owner, .tree = tree, .key = {key, key_len}, .need = txn_memtable_version_encoded(&change->writer, change->op, change->shown, change->has_lsn, change->lsn)}}; - txn_memtable_table_alone_t alone = {change, result}; + txn_memtable_table_alone_t alone = {change, has_roomed, roomed, result}; TxnMemtableTableReservation none = {0}; bool has_unstored; TxnMemtableTableReservation unstored; return txn_memtable_table_prepared(io, aims, 1, false, none, txn_memtable_table_alone_inside, &alone, NULL, - &has_unstored, &unstored, error); + &has_unstored, &unstored, NULL, error); } /* @@ -1551,10 +1942,13 @@ txn_memtable_table_alone_State(HostIoIo *io, HostSharedRecord owner, bool has_wh const uint8 *key, size_t key_len, const TxnMemtableVersionChange *change, TxnMemtableVersionHandle *result, HostError *error) { + TxnStatusRoomed unroomed = {0}; + const TxnMemtableTableTreeOps *tree = !has_which ? &TXN_MEMTABLE_TABLE_ROWS_TREE_OPS : which == TXN_MEMTABLE_MAP_MAP_FREE ? &TXN_MEMTABLE_MAP_FREE_TREE_OPS : &TXN_MEMTABLE_MAP_VALUES_TREE_OPS; - return txn_memtable_table_alone(tree, io, owner, key, key_len, change, result, error); + return txn_memtable_table_alone(tree, io, owner, key, key_len, change, false, unroomed, result, + error); } /* @@ -1566,7 +1960,10 @@ bool txn_memtable_table_push(HostIoIo *io, HostSharedRecord state, const uint8 *key, size_t key_len, const TxnMemtableVersionChange *change, TxnMemtableVersionHandle *result, HostError *error) { - return txn_memtable_table_alone(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, state, key, key_len, change, result, error); + TxnStatusRoomed unroomed = {0}; + + return txn_memtable_table_alone(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, state, key, key_len, change, false, + unroomed, result, error); } /* TxnMemtableTableOnePage is an iterator over one owned page, the pages a first range's lock owns at its link. */ @@ -1636,6 +2033,19 @@ typedef struct txn_memtable_table_first_frame_0_t bool with_live; HostSharedGuard guard; bool guard_live; + + /* + * given is the first page the prepare's allocation took, until the + * memtable takes it. + */ + HostDeltaOwned given; + bool has_given; + + /* + * refused is whether the delta refused a record, with its refusal. + */ + bool *refused; + HostDeltaFull *full; } txn_memtable_table_first_frame_0_t; /* txn_memtable_table_first_pad_0 is first's pad: the guard, the With, the first page, the memtable, the range, and the pages, while live. */ @@ -1655,6 +2065,11 @@ txn_memtable_table_first_pad_0(void *frame_) host_delta_With_drop(&frame->with); } txn_memtable_table_OnePage_drop(&frame->page); + if (frame->has_given) + { + frame->has_given = false; + host_delta_Owned_drop(&frame->given); + } if (frame->memtable_live) { frame->memtable_live = false; @@ -1674,57 +2089,45 @@ txn_memtable_table_first_pad_0(void *frame_) } /* - * txn_memtable_table_first_made takes the first range's page and makes the - * range and the memtable, waiting while the delta can't give them, answering - * false on an error. + * txn_memtable_table_first_made takes the first range's page it was given and + * makes the range and the memtable in one try, answering the delta's refusal + * of a record, with what it took given back by the pad, rather than waiting, + * and false on an error. */ static bool -txn_memtable_table_first_made(txn_memtable_table_first_frame_0_t *frame, HostDeltaPageSize size) +txn_memtable_table_first_made(txn_memtable_table_first_frame_0_t *frame) { - for (;;) - { - bool allocated; - HostDeltaFull full; + bool allocated; - if (!host_delta_alloc(frame->io, &size, 1, &allocated, &frame->pages, &full, frame->error)) - return false; - if (!allocated) - { - if (!txn_memtable_table_waited(frame->io, full, false, frame->error)) - return false; - continue; - } - frame->pages_live = true; - if (!host_delta_make(frame->io, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS, &allocated, &frame->range, &full, - frame->error)) - return false; - if (!allocated) - { - txn_memtable_table_first_pad_0(frame); - if (!txn_memtable_table_waited(frame->io, full, false, frame->error)) - return false; - continue; - } - frame->range_live = true; - if (!host_delta_make(frame->io, &TXN_MEMTABLE_RANGE_MEMTABLE_AGED_OPS, &allocated, &frame->memtable, &full, - frame->error)) - return false; - if (!allocated) - { - txn_memtable_table_first_pad_0(frame); - if (!txn_memtable_table_waited(frame->io, full, false, frame->error)) - return false; - continue; - } - frame->memtable_live = true; - frame->page.has = std_vec_pop(&frame->pages, &frame->page.page, sizeof(HostDeltaOwned)); - if (!frame->page.has) - { - *frame->error = txn_memtable_table_internal("an allocation gave no page"); - return false; - } + std_vec_with_capacity_Checked(&frame->pages, 1, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + std_vec_push_Checked(&frame->pages, &frame->given, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + frame->has_given = false; + frame->pages_live = true; + if (!host_delta_make(frame->io, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS, &allocated, &frame->range, frame->full, + frame->error)) + return false; + if (!allocated) + { + *frame->refused = true; return true; } + frame->range_live = true; + if (!host_delta_make(frame->io, &TXN_MEMTABLE_RANGE_MEMTABLE_AGED_OPS, &allocated, &frame->memtable, frame->full, + frame->error)) + return false; + if (!allocated) + { + *frame->refused = true; + return true; + } + frame->memtable_live = true; + frame->page.has = std_vec_pop(&frame->pages, &frame->page.page, sizeof(HostDeltaOwned)); + if (!frame->page.has) + { + *frame->error = txn_memtable_table_internal("an allocation gave no page"); + return false; + } + return true; } /* txn_memtable_table_filled fills one of a made record's lists with its one entry while it is unlinked. */ @@ -1743,96 +2146,222 @@ txn_memtable_table_filled(const HostDeltaMade *made, const HostSharedShapeOps *s return true; } -/* txn_memtable_table_first_body_0 is first's body. */ +/* + * txn_memtable_table_first_linked makes, once its records are taken, the + * memtable on its page: the empty key as its range's first and each record's + * list with its one entry; then, under the owner's lock, one link of the + * memtable into the tree's word makes them live together if the word still + * names none. It answers false on an error, the pad giving back what isn't + * linked. + */ +static bool +txn_memtable_table_first_linked(txn_memtable_table_first_frame_0_t *frame) +{ + uint64 location; + HostDeltaPageSize size = txn_memtable_table_first_size(); + uint64 name; + HostSharedOp op = {.tag = HOST_SHARED_OP_EXCHANGE}; + StdIterator pages; + HostSharedPast word; + + if (!txn_memtable_index_begin(&TXN_MEMTABLE_RANGE_RANGE_KEYED_OPS, &frame->range, &frame->page.page, 0, NULL, 0, + frame->error)) + return false; + name = host_delta_page_Page_value(host_delta_Owned_name(&frame->page.page)); + if (!txn_memtable_table_filled(&frame->range, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS.shape, TXN_MEMTABLE_INDEX_PAGES, + name, frame->error)) + return false; + op.u.exchange.f1 = name; + host_delta_Made_atomic(&frame->range, TXN_MEMTABLE_RANGE_RANGE_START, op); + op.u.exchange.f1 = (uint64) size.words; + host_delta_Made_atomic(&frame->range, TXN_MEMTABLE_RANGE_RANGE_NEWEST, op); + if (!txn_memtable_table_filled(&frame->memtable, &TXN_MEMTABLE_RANGE_MEMTABLE_AGED_OPS.shape, + TXN_MEMTABLE_RANGE_RANGES, host_delta_Made_location(&frame->range), frame->error)) + return false; + frame->with = host_delta_With_Default_default(); + frame->with_live = true; + pages = (StdIterator) + { + &TXN_MEMTABLE_TABLE_ONE_PAGE_ITERATOR_OPS, &frame->page + }; + frame->range_live = false; + if (!host_delta_With_record(&frame->with, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS, TXN_MEMTABLE_RANGE_RANGE_LOCK, + &frame->range, pages, frame->error) || + !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-memtable-first", frame->error) || + !host_shared_Record_lock(frame->owner, HOST_IO_IO_TAKES_SHORT(frame->io), frame->tree->lock, &frame->guard, + frame->error)) + return false; + frame->guard_live = true; + word = host_shared_Record_names_past(frame->owner, frame->tree->word); + if (!host_shared_Past_load(&word, &location)) + { + HostDeltaNames names = {&HOST_DELTA_PAST_NAMES_OPS, &word}; + + frame->memtable_live = false; + frame->with_live = false; + host_delta_Made_link(&frame->memtable, &TXN_MEMTABLE_RANGE_MEMTABLE_AGED_OPS, names, &frame->with); + } + return true; +} + +/* + * txn_memtable_table_prepared_firsts takes, for every memtable the try's + * trees lack, its range's and memtable's records on the first page the try's + * allocation took for it, all before any memtable is made, so the refusal of + * any one's records gives every record and page taken back and leaves no + * memtable. It answers false on an error. + */ +static bool +txn_memtable_table_prepared_firsts(txn_memtable_table_prepared_frame_0_t *frame, bool *refused, HostDeltaFull *full) +{ + size_t missing = frame->missing.len; + + std_vec_new(&frame->firsts, sizeof(txn_memtable_table_first_frame_0_t), _Alignof(txn_memtable_table_first_frame_0_t)); + frame->firsts_live = true; + if (!std_vec_try_reserve_exact(&frame->firsts, missing, sizeof(txn_memtable_table_first_frame_0_t), + _Alignof(txn_memtable_table_first_frame_0_t))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + for (size_t i = 0; i < missing; i++) + { + const TxnMemtableTableAim *aim = &frame->aims[STD_VEC_AT(&frame->missing, size_t, missing - 1 - i)]; + txn_memtable_table_first_frame_0_t first = {.tree = aim->tree, .io = frame->io, .owner = aim->owner, + .error = frame->error, .refused = refused, .full = full, + .given = STD_VEC_AT(&frame->pages, HostDeltaOwned, frame->pages.len - 1), .has_given = true}; + uint64 location; + + frame->pages.len--; + std_vec_push_Checked(&frame->firsts, &first, sizeof(first), _Alignof(txn_memtable_table_first_frame_0_t)); + if (txn_memtable_table_memtable(aim->tree, aim->owner, &location)) + continue; + if (!txn_memtable_table_first_made(&STD_VEC_AT(&frame->firsts, txn_memtable_table_first_frame_0_t, i))) + return false; + if (*refused) + { + txn_memtable_table_prepared_firsts_drop(frame); + return true; + } + } + return true; +} + +/* + * txn_memtable_table_prepared_linked makes and links each memtable whose + * records the try took, then hands its aim's push off to its key's range, + * answering some false when a range gave out and false on an error. + */ +static bool +txn_memtable_table_prepared_linked(txn_memtable_table_prepared_frame_0_t *frame, bool *some) +{ + size_t missing = frame->missing.len; + + *some = true; + for (size_t i = 0; i < frame->firsts.len && *some; i++) + { + txn_memtable_table_first_frame_0_t *first = &STD_VEC_AT(&frame->firsts, txn_memtable_table_first_frame_0_t, i); + size_t k = STD_VEC_AT(&frame->missing, size_t, missing - 1 - i); + const TxnMemtableTableAim *aim = &frame->aims[k]; + bool linked = !first->memtable_live || txn_memtable_table_first_linked(first); + + txn_memtable_table_first_pad_0(first); + if (!linked || + !txn_memtable_table_handed(aim->tree, frame->io, &frame->prepared.hold, &frame->mine, aim->owner, + aim->key.ptr, aim->key.len, some, &STD_VEC_AT(&frame->ats, TxnMemtableTableAt, k), + frame->error)) + return false; + } + txn_memtable_table_prepared_firsts_drop(frame); + frame->missing.len = 0; + return true; +} + +/* txn_memtable_table_prepared_firsts_drop gives back what each unlinked memtable's records and first page hold, while live. */ +static void +txn_memtable_table_prepared_firsts_drop(txn_memtable_table_prepared_frame_0_t *frame) +{ + if (!frame->firsts_live) + return; + frame->firsts_live = false; + for (size_t i = 0; i < frame->firsts.len; i++) + txn_memtable_table_first_pad_0(&STD_VEC_AT(&frame->firsts, txn_memtable_table_first_frame_0_t, i)); + std_vec_free_Checked(&frame->firsts, sizeof(txn_memtable_table_first_frame_0_t), + _Alignof(txn_memtable_table_first_frame_0_t)); +} + +#ifdef DOLTGRES_TEST_TXN +/* txn_memtable_table_first_body_0 is made's body. */ static void txn_memtable_table_first_body_0(void *frame_) { txn_memtable_table_first_frame_0_t *frame = frame_; uint64 location; - HostDeltaPageSize size; - uint64 name; - HostSharedOp op = {.tag = HOST_SHARED_OP_EXCHANGE}; - StdIterator pages; - HostSharedPast word; if (txn_memtable_table_memtable(frame->tree, frame->owner, &location)) { frame->ok = true; return; } - size = txn_memtable_pages_size(false, 0, 0, txn_memtable_index_least(NULL, 0).words); - if (!txn_memtable_table_first_made(frame, size) || - !txn_memtable_index_begin(&TXN_MEMTABLE_RANGE_RANGE_KEYED_OPS, &frame->range, &frame->page.page, 0, NULL, 0, - frame->error)) - { - txn_memtable_table_first_pad_0(frame); - return; - } - name = host_delta_page_Page_value(host_delta_Owned_name(&frame->page.page)); - if (!txn_memtable_table_filled(&frame->range, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS.shape, TXN_MEMTABLE_INDEX_PAGES, - name, frame->error)) - { - txn_memtable_table_first_pad_0(frame); - return; - } - op.u.exchange.f1 = name; - host_delta_Made_atomic(&frame->range, TXN_MEMTABLE_RANGE_RANGE_START, op); - op.u.exchange.f1 = (uint64) size.words; - host_delta_Made_atomic(&frame->range, TXN_MEMTABLE_RANGE_RANGE_NEWEST, op); - if (!txn_memtable_table_filled(&frame->memtable, &TXN_MEMTABLE_RANGE_MEMTABLE_AGED_OPS.shape, - TXN_MEMTABLE_RANGE_RANGES, host_delta_Made_location(&frame->range), frame->error)) - { - txn_memtable_table_first_pad_0(frame); - return; - } - frame->with = host_delta_With_Default_default(); - frame->with_live = true; - pages = (StdIterator) - { - &TXN_MEMTABLE_TABLE_ONE_PAGE_ITERATOR_OPS, &frame->page - }; - frame->range_live = false; - if (!host_delta_With_record(&frame->with, &TXN_MEMTABLE_RANGE_RANGE_AGED_OPS, TXN_MEMTABLE_RANGE_RANGE_LOCK, - &frame->range, pages, frame->error) || - !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-memtable-first", frame->error) || - !host_shared_Record_lock(frame->owner, HOST_IO_IO_TAKES_SHORT(frame->io), frame->tree->lock, &frame->guard, - frame->error)) - { - txn_memtable_table_first_pad_0(frame); - return; - } - frame->guard_live = true; - word = host_shared_Record_names_past(frame->owner, frame->tree->word); - if (!host_shared_Past_load(&word, &location)) - { - HostDeltaNames names = {&HOST_DELTA_PAST_NAMES_OPS, &word}; - - frame->memtable_live = false; - frame->with_live = false; - host_delta_Made_link(&frame->memtable, &TXN_MEMTABLE_RANGE_MEMTABLE_AGED_OPS, names, &frame->with); - } - frame->ok = true; + frame->ok = txn_memtable_table_first_made(frame) && (*frame->refused || txn_memtable_table_first_linked(frame)); txn_memtable_table_first_pad_0(frame); } /* - * txn_memtable_table_first makes a tree's memtable as its first writer: with - * no hold and no lock it takes the range's first page, makes the range and - * the memtable, writes the empty key there as the range's first, and starts - * each record's list with its one entry; then, under the owner's lock, one - * link of the memtable into the tree's word makes them live together if the - * word still names none, and otherwise its drops give everything back. It - * does nothing for a tree whose word names a memtable. + * txn_memtable_table_made_on makes a tree's memtable as its first writer in + * one try, on the first page a prepare's allocation took with its reserves: + * with no lock it makes the range and the memtable on the page, writes the + * empty key there as the range's first, and starts each record's list with + * its one entry; then, under the owner's lock, one link of the memtable into + * the tree's word makes them live together if the word still names none, + * and otherwise its drops give everything back. It gives the page back for a + * tree whose word names a memtable, and answers the delta's refusal of a + * record, with what it took given back, rather than waiting. */ -bool -txn_memtable_table_first(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, HostError *error) +static bool +txn_memtable_table_made_on(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, + HostDeltaOwned page, bool *refused, HostDeltaFull *full, HostError *error) { - txn_memtable_table_first_frame_0_t frame = {.tree = tree, .io = io, .owner = owner, .error = error}; + txn_memtable_table_first_frame_0_t frame = {.tree = tree, .io = io, .owner = owner, .error = error, + .refused = refused, .full = full, .given = page, .has_given = true}; + *refused = false; std_landing(&frame, txn_memtable_table_first_body_0, txn_memtable_table_first_pad_0); + if (frame.has_given) + host_delta_Owned_drop(&frame.given); return frame.ok; } +/* txn_memtable_table_first makes a tree's memtable as its first writer, as made_on does on a page it takes, waiting at the delta's budget when it refuses. */ +bool +txn_memtable_table_first(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, HostError *error) +{ + for (;;) + { + HostDeltaPageSize size = txn_memtable_table_first_size(); + bool allocated; + bool refused = true; + StdVec pages; + HostDeltaFull full; + + if (!host_delta_alloc(io, &size, 1, &allocated, &pages, &full, error)) + return false; + if (allocated) + { + HostDeltaOwned page = STD_VEC_AT(&pages, HostDeltaOwned, 0); + + std_vec_free_Checked(&pages, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + if (!txn_memtable_table_made_on(tree, io, owner, page, &refused, &full, error)) + return false; + } + if (!refused) + return true; + if (!txn_memtable_table_waited(io, full, false, error)) + return false; + } +} +#endif + /* txn_memtable_table_abandoned_frame_0_t is abandoned's state across its pad: the hold and the range's guard, while live. */ typedef struct txn_memtable_table_abandoned_frame_0_t { @@ -2125,6 +2654,7 @@ typedef struct txn_memtable_table_removed_frame_0_t bool names_live; StdVec counted; bool counted_live; + StdVec *given; } txn_memtable_table_removed_frame_0_t; /* txn_memtable_table_removed_take_pad drops the lists of one range's take, while live. */ @@ -2225,7 +2755,8 @@ txn_memtable_table_removed_keys(txn_memtable_table_removed_frame_0_t *frame, Txn if (some) { if (!txn_memtable_version_removed(open, &frame->hold, node, key->handles, key->handles_len, taken->ptr, - taken->len, &emptied, &frame->touched, &bytes, frame->error)) + taken->len, frame->given, &emptied, &frame->touched, &bytes, + frame->error)) return false; frame->touched_live = true; if (bytes > 0) @@ -2566,24 +3097,43 @@ txn_memtable_table_removed_body_0(void *frame_) txn_memtable_table_removed_pad_0(frame); } +/* + * txn_memtable_table_removing is removed_all's walk, appending to given the + * top-level xid of each version it takes out that its push counted. + */ +static bool +txn_memtable_table_removing(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, + const TxnMemtableTableRemoving *keys, size_t keys_len, StdVec *result, StdVec *given, + HostError *error) +{ + txn_memtable_table_removed_frame_0_t frame = {.tree = tree, .io = io, .owner = owner, .keys = keys, + .keys_len = keys_len, .result = result, .given = given, .error = error}; + + std_landing(&frame, txn_memtable_table_removed_body_0, txn_memtable_table_removed_pad_0); + return frame.ok; +} + /* * txn_memtable_table_removed_all takes versions out of the chains of keys * given in byte order, as removed does for one key, each range taking out * every key of the run it holds under one take of its exclusive lock, with * one sweep of its dropped pages and one compaction after, as a spill takes * a transaction's keys out of its memtable. It answers, for each key at each - * handle's place, whether this removal took it out. + * handle's place, whether this removal took it out. It gives back no count, + * since a spill moves the versions to its file, whose own count names them. */ bool txn_memtable_table_removed_all(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, const TxnMemtableTableRemoving *keys, size_t keys_len, StdVec *result, HostError *error) { - txn_memtable_table_removed_frame_0_t frame = {.tree = tree, .io = io, .owner = owner, .keys = keys, - .keys_len = keys_len, .result = result, .error = error}; + StdVec given; + bool ok; - std_landing(&frame, txn_memtable_table_removed_body_0, txn_memtable_table_removed_pad_0); - return frame.ok; + std_vec_new(&given, sizeof(uint64), _Alignof(uint64)); + ok = txn_memtable_table_removing(tree, io, owner, keys, keys_len, result, &given, error); + std_vec_free_Checked(&given, sizeof(uint64), _Alignof(uint64)); + return ok; } /* @@ -2592,8 +3142,9 @@ txn_memtable_table_removed_all(const TxnMemtableTableTreeOps *tree, HostIoIo *io * range's exclusive lock: the versions leave the chain, a key left with none * leaves the index, and its dropped pages and its compaction are as * removed_all's. A key the tree doesn't hold, and a handle its chain - * doesn't, are passed over. It answers, at each handle's place, whether this - * removal took it out. + * doesn't, are passed over. Each version it takes out that its push counted + * gives its count back at its top-level transaction's entry, after the take. + * It answers, at each handle's place, whether this removal took it out. */ bool txn_memtable_table_removed(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, @@ -2602,8 +3153,24 @@ txn_memtable_table_removed(const TxnMemtableTableTreeOps *tree, HostIoIo *io, Ho { TxnMemtableTableRemoving keys[1] = {{{key, key_len}, handles, handles_len}}; StdVec taken; + StdVec given; + bool ok; + + std_vec_new(&given, sizeof(uint64), _Alignof(uint64)); + ok = txn_memtable_table_removing(tree, io, owner, keys, 1, &taken, &given, error); + for (size_t i = 0; ok && i < given.len;) + { + HostXid top = {STD_VEC_AT(&given, uint64, i)}; + uint64 count = 0; + bool has; + uint64 added; - if (!txn_memtable_table_removed_all(tree, io, owner, keys, 1, &taken, error)) + for (; i < given.len && STD_VEC_AT(&given, uint64, i) == top.f0; i++) + count++; + ok = txn_status_add(io, top, std_wrapping_sub_u64(0, count), &has, &added, error); + } + std_vec_free_Checked(&given, sizeof(uint64), _Alignof(uint64)); + if (!ok) return false; if (!std_vec_pop(&taken, result, sizeof(StdVec))) std_vec_new(result, sizeof(bool), _Alignof(bool)); @@ -2612,13 +3179,12 @@ txn_memtable_table_removed(const TxnMemtableTableTreeOps *tree, HostIoIo *io, Ho return true; } -/* txn_memtable_table_Pruning_t is a chain entry pruned reads: a version's handle, its writer, its hint, and whether it is a lock's. */ +/* txn_memtable_table_Pruning_t is a chain entry pruned reads: a version's handle, its writer, and its hint. */ typedef struct txn_memtable_table_Pruning_t { TxnMemtableVersionHandle handle; TxnMemtableVersionWriter writer; TxnMemtableVersionHint hint; - bool lock; } txn_memtable_table_Pruning_t; /* txn_memtable_table_pruned_t is pruned's reading's captures: the key and the chain read. */ @@ -2629,15 +3195,6 @@ typedef struct txn_memtable_table_pruned_t StdVec chain; } txn_memtable_table_pruned_t; -/* txn_memtable_table_is_lock reports whether a version's op is a lock's, which counts nothing. */ -static bool -txn_memtable_table_is_lock(const TxnMemtableVersionRecord *record) -{ - TxnMemtableVersionOpRefTag op = txn_memtable_version_Record_op(record).tag; - - return op == TXN_MEMTABLE_VERSION_OP_REF_LOCK || op == TXN_MEMTABLE_VERSION_OP_REF_LOCK_ROW; -} - /* txn_memtable_table_pruned_reading reads a key's chain's writers under its range's shared lock. */ static bool txn_memtable_table_pruned_reading(void *captures, const HostDeltaHold *hold, TxnMemtableTableAt at, @@ -2673,7 +3230,6 @@ txn_memtable_table_pruned_reading(void *captures, const HostDeltaHold *hold, Txn entry.handle = version.handle; entry.writer = txn_memtable_version_Record_writer(&record); entry.hint = txn_memtable_version_Version_hint(&version); - entry.lock = txn_memtable_table_is_lock(&record); txn_memtable_version_Version_drop(&version); std_vec_push_Checked(&pruned->chain, &entry, sizeof(entry), _Alignof(txn_memtable_table_Pruning_t)); } @@ -2794,27 +3350,6 @@ txn_memtable_table_pruned_body_0(void *frame_) return; } frame->taken_live = true; - for (size_t k = 0; k < kept && k < frame->taken.len; k++) - { - const txn_memtable_table_Pruning_t *entry = &STD_VEC_AT(&frame->pruned.chain, - txn_memtable_table_Pruning_t, k); - TxnMemtableVersionWriter none = TXN_MEMTABLE_VERSION_NO_TRANSACTION; - - if (!STD_VEC_AT(&frame->taken, bool, k)) - continue; - if (frame->tree->counted && !entry->lock && !txn_memtable_version_Writer_PartialEq_eq(&entry->writer, &none)) - { - HostXid top = {entry->writer.has_top ? entry->writer.top : entry->writer.xid}; - bool has; - uint64 added; - - if (!txn_status_add(frame->io, top, PG_UINT64_MAX, &has, &added, frame->error)) - { - txn_memtable_table_pruned_pad_0(frame); - return; - } - } - } frame->ok = true; txn_memtable_table_pruned_pad_0(frame); } @@ -2828,10 +3363,9 @@ txn_memtable_table_pruned_body_0(void *frame_) * any a concurrent removal took first. A committed version is the fold's, * never pruned here. A top-level transaction's abort names its xid as * aborting, and every version it or one of its subtransactions wrote is - * aborted with no question. Each version but a lock's this removal took out - * gives its count back at its top-level transaction's entry, so a rolled-back - * savepoint's versions leave a committed transaction's count as they found - * it. + * aborted with no question. The remover gives back the count of each counted + * version it took out, so a rolled-back savepoint's versions leave a + * committed transaction's count as they found it. */ bool txn_memtable_table_pruned(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, @@ -2844,28 +3378,6 @@ txn_memtable_table_pruned(const TxnMemtableTableTreeOps *tree, HostIoIo *io, Hos return frame.ok; } -/* - * txn_memtable_table_uncounted takes back the add a version's push made at - * its writer's entry, for a version taken out of the delta: every version - * but a lock's, as the finishes count them, at its top-level transaction's - * entry; a writer whose entry is gone takes nothing back. - */ -bool -txn_memtable_table_uncounted(HostIoIo *io, const TxnMemtableVersionVersion *version, HostError *error) -{ - TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); - TxnMemtableVersionWriter writer; - HostXid top; - bool has; - uint64 added; - - if (txn_memtable_table_is_lock(&record)) - return true; - writer = txn_memtable_version_Record_writer(&record); - top.f0 = writer.has_top ? writer.top : writer.xid; - return txn_status_add(io, top, PG_UINT64_MAX, &has, &added, error); -} - /* TxnMemtableTableEmptyPage is a page dropped_pages found with no live record: its entry, its name, and its page. */ typedef struct TxnMemtableTableEmptyPage { @@ -3747,6 +4259,49 @@ txn_memtable_table_chain(const TxnMemtableTableTreeOps *tree, HostIoIo *io, cons return false; } +/* txn_memtable_table_versions_read_t is versions' read's captures: its arguments and the versions it lends. */ +typedef struct txn_memtable_table_versions_read_t +{ + HostIoIo *io; + const HostDeltaHold *hold; + HostSharedRecord state; + const uint8 *key; + size_t key_len; + StdVec *result; +} txn_memtable_table_versions_read_t; + +/* txn_memtable_table_versions_dropped drops the versions a read of versions lent. */ +static void +txn_memtable_table_versions_dropped(void *captures) +{ + txn_memtable_table_versions_read_t *c = captures; + + std_vec_clear(c->result, sizeof(TxnMemtableVersionVersion), txn_memtable_table_Version_drop); + std_vec_free_Checked(c->result, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); +} + +/* txn_memtable_table_versions_read is versions' read: the key's chain in the memtable, then its versions in the spills. */ +static bool +txn_memtable_table_versions_read(void *captures, uint64 *listed, HostError *error) +{ + txn_memtable_table_versions_read_t *c = captures; + StdVec spilled; + + if (!txn_memtable_table_chain(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, c->io, c->hold, c->state, c->key, c->key_len, + c->result, error)) + return false; + if (!txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, c->io, c->hold, c->state, TXN_RUNS_KIND_SPILLED_ROWS, c->key, + c->key_len, listed, &spilled, error)) + { + txn_memtable_table_versions_dropped(c); + return false; + } + std_vec_extend_from_slice_Checked(c->result, spilled.ptr, spilled.len, sizeof(TxnMemtableVersionVersion), + _Alignof(TxnMemtableVersionVersion)); + std_vec_free_Checked(&spilled, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + return true; +} + /* * txn_memtable_table_versions lends a key's versions in a storage's rows, * newest first, the memtable's and then its spills', for a caller's hold: @@ -3758,21 +4313,10 @@ bool txn_memtable_table_versions(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, const uint8 *key, size_t key_len, StdVec *result, HostError *error) { - StdVec spilled; + txn_memtable_table_versions_read_t captures = {io, hold, state, key, key_len, result}; - if (!txn_memtable_table_chain(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, hold, state, key, key_len, result, error)) - return false; - if (!txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, io, hold, state, TXN_RUNS_KIND_SPILLED_ROWS, key, key_len, - &spilled, error)) - { - std_vec_clear(result, sizeof(TxnMemtableVersionVersion), txn_memtable_table_Version_drop); - std_vec_free_Checked(result, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); - return false; - } - std_vec_extend_from_slice_Checked(result, spilled.ptr, spilled.len, sizeof(TxnMemtableVersionVersion), - _Alignof(TxnMemtableVersionVersion)); - std_vec_free_Checked(&spilled, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); - return true; + return txn_spill_taken(&TXN_SPILL_STATE_HOME_OPS, io, state, txn_memtable_table_versions_read, + txn_memtable_table_versions_dropped, &captures, error); } /* txn_memtable_table_bytes_drop drops a Vec in a list of them. */ @@ -3799,7 +4343,7 @@ txn_memtable_table_CopiedKey_drop(void *self_) std_vec_free_Checked(&self->key, 1, 1); std_vec_clear(&self->records, sizeof(StdVec), txn_memtable_table_bytes_drop); std_vec_free_Checked(&self->records, sizeof(StdVec), _Alignof(StdVec)); - std_vec_free_Checked(&self->handles, sizeof(txn_TupleHandleU64_t), _Alignof(txn_TupleHandleU64_t)); + std_vec_free_Checked(&self->handles, sizeof(txn_TupleHandleU64Bool_t), _Alignof(txn_TupleHandleU64Bool_t)); } /* txn_memtable_table_copied_reading_t is copied's reading's captures: the walk's place, whether it hands out killed versions, and the copy. */ @@ -3825,15 +4369,16 @@ txn_memtable_table_copied_key(txn_memtable_table_copied_reading_t *reading, cons { std_slice_to_vec_Checked(©.key, key.ptr, key.len, 1, 1); std_vec_new(©.records, sizeof(StdVec), _Alignof(StdVec)); - std_vec_new(©.handles, sizeof(txn_TupleHandleU64_t), _Alignof(txn_TupleHandleU64_t)); + std_vec_new(©.handles, sizeof(txn_TupleHandleU64Bool_t), _Alignof(txn_TupleHandleU64Bool_t)); for (size_t k = 0; k < chain.len; k++) { const TxnMemtableVersionVersion *version = &STD_VEC_AT(&chain, TxnMemtableVersionVersion, k); TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); TxnMemtableVersionWriter writer = txn_memtable_version_Record_writer(&record); - txn_TupleHandleU64_t handle = {version->handle, writer.has_top ? writer.top : writer.xid}; + txn_TupleHandleU64Bool_t handle = {version->handle, writer.has_top ? writer.top : writer.xid, + txn_memtable_version_Version_killed(version)}; - std_vec_push_Checked(©.handles, &handle, sizeof(handle), _Alignof(txn_TupleHandleU64_t)); + std_vec_push_Checked(©.handles, &handle, sizeof(handle), _Alignof(txn_TupleHandleU64Bool_t)); if (reading->killed || !txn_memtable_version_Version_killed(version)) { StdVec stored; @@ -3895,7 +4440,7 @@ typedef struct txn_memtable_table_copied_frame_0_t HostSharedRecord owner; bool killed; bool (*each) (void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *records, - size_t records_len, const txn_TupleHandleU64_t *handles, size_t handles_len, + size_t records_len, const txn_TupleHandleU64Bool_t *handles, size_t handles_len, HostError *error); void *captures; HostError *error; @@ -4006,12 +4551,12 @@ txn_memtable_table_copied_body_0(void *frame_) * txn_memtable_table_copied hands out every key of a tree's memtable that * has a version, in byte order, with its versions' records newest first, as * a checkpoint's runs write them, a speculative insert its writer killed - * among them only when killed says so, and every version's handle and its - * writer's top-level xid, newest first, the killed included. Each step - * copies one range's keys from the walk's place up to the key the step stops - * at, the next entry's first key, under the range's shared lock, and hands - * them out with the lock released, so no lock is held across the caller's - * writes; it goes on from that key. A range that splits or is emptied and + * among them only when killed says so, and every version's handle, its + * writer's top-level xid, and whether it is killed, newest first, the killed + * included. Each step copies one range's keys from the walk's place up to + * the key the step stops at, the next entry's first key, under the range's + * shared lock, and hands them out with the lock released, so no lock is held + * across the caller's writes; it goes on from that key. A range that splits or is emptied and * retired between the steps leaves no key skipped or handed out twice. * Copying allocates, since a range's records must outlive its lock. */ @@ -4019,7 +4564,7 @@ bool txn_memtable_table_copied(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, bool killed, bool (*each) (void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *records, size_t records_len, - const txn_TupleHandleU64_t *handles, size_t handles_len, HostError *error), + const txn_TupleHandleU64Bool_t *handles, size_t handles_len, HostError *error), void *captures, HostError *error) { txn_memtable_table_copied_frame_0_t frame = {.tree = tree, .io = io, .owner = owner, .killed = killed, @@ -4040,7 +4585,7 @@ typedef struct txn_memtable_table_walk_t /* txn_memtable_table_walk_each hands a key with records to the caller's each, passing over one with none. */ static bool txn_memtable_table_walk_each(void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *records, - size_t records_len, const txn_TupleHandleU64_t *handles, size_t handles_len, + size_t records_len, const txn_TupleHandleU64Bool_t *handles, size_t handles_len, HostError *error) { txn_memtable_table_walk_t *walk = captures; @@ -5526,28 +6071,51 @@ txn_memtable_table_nearest(HostIoIo *io, const HostDeltaHold *hold, HostSharedRe return frame.ok; } -/* - * txn_memtable_table_next lends the nearest key past a key, or before it - * going backward, or the first or the last with none, that a storage's rows - * hold a version of, in the memtable or its spills, with its versions newest - * first, the memtable's and then the spills' for a key both hold, for a - * caller's hold; it answers nothing past the last. - */ -bool -txn_memtable_table_next(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, bool has_after, - const uint8 *after, size_t after_len, TxnMemtableTableDirection way, bool *some, - txn_TupleVecU8VecVersion_t *result, HostError *error) +/* txn_memtable_table_next_read_t is next's read's captures: its arguments and the key it lends. */ +typedef struct txn_memtable_table_next_read_t +{ + HostIoIo *io; + const HostDeltaHold *hold; + HostSharedRecord state; + bool has_after; + const uint8 *after; + size_t after_len; + TxnMemtableTableDirection way; + bool *some; + txn_TupleVecU8VecVersion_t *result; +} txn_memtable_table_next_read_t; + +/* txn_memtable_table_next_dropped drops the key a read of next lent. */ +static void +txn_memtable_table_next_dropped(void *captures) +{ + txn_memtable_table_next_read_t *c = captures; + + if (*c->some) + txn_memtable_table_Versions_drop(c->result); + *c->some = false; +} + +/* txn_memtable_table_next_read is next's read: the memtable's nearest key, then the spills', the nearer lent. */ +static bool +txn_memtable_table_next_read(void *captures, uint64 *listed, HostError *error) { + txn_memtable_table_next_read_t *c = captures; + HostIoIo *io = c->io; + TxnMemtableTableDirection way = c->way; + bool *some = c->some; + txn_TupleVecU8VecVersion_t *result = c->result; bool has_delta; txn_TupleVecU8VecVersion_t delta; bool has_spilled; txn_TupleVecU8VecVersion_t spilled; *some = false; - if (!txn_memtable_table_nearest(io, hold, state, has_after, after, after_len, way, &has_delta, &delta, error)) + if (!txn_memtable_table_nearest(io, c->hold, c->state, c->has_after, c->after, c->after_len, way, &has_delta, + &delta, error)) return false; - if (!txn_spill_next(&TXN_SPILL_STATE_HOME_OPS, io, hold, state, TXN_RUNS_KIND_SPILLED_ROWS, has_after, after, - after_len, way, &has_spilled, &spilled, error)) + if (!txn_spill_next(&TXN_SPILL_STATE_HOME_OPS, io, c->hold, c->state, TXN_RUNS_KIND_SPILLED_ROWS, c->has_after, + c->after, c->after_len, way, listed, &has_spilled, &spilled, error)) { if (has_delta) txn_memtable_table_Versions_drop(&delta); @@ -5588,6 +6156,25 @@ txn_memtable_table_next(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecor return true; } +/* + * txn_memtable_table_next lends the nearest key past a key, or before it + * going backward, or the first or the last with none, that a storage's rows + * hold a version of, in the memtable or its spills, with its versions newest + * first, the memtable's and then the spills' for a key both hold, for a + * caller's hold; it answers nothing past the last. + */ +bool +txn_memtable_table_next(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, bool has_after, + const uint8 *after, size_t after_len, TxnMemtableTableDirection way, bool *some, + txn_TupleVecU8VecVersion_t *result, HostError *error) +{ + txn_memtable_table_next_read_t captures = {io, hold, state, has_after, after, after_len, way, some, result}; + + *some = false; + return txn_spill_taken(&TXN_SPILL_STATE_HOME_OPS, io, state, txn_memtable_table_next_read, + txn_memtable_table_next_dropped, &captures, error); +} + /* The prepare whose closure returns nothing, which every crate that takes it declares and table.c defines. */ #define TXN_MEMTABLE_TABLE_RS Unit #define TXN_MEMTABLE_TABLE_RETURNS bool @@ -8010,7 +8597,8 @@ txn_memtable_table_tests_raw_body_0(void *frame_) change.op = &frame->op; std_vec_new(&frame->buffer, 1, 1); frame->buffer_live = true; - if (!txn_memtable_version_push(&open, &frame->hold, frame->key.ptr, frame->key.len, &change, &frame->has_fresh, + if (!txn_memtable_version_push(&open, &frame->hold, frame->key.ptr, frame->key.len, &change, false, + &frame->has_fresh, &frame->fresh, &frame->spare, &frame->buffer, &pushed, frame->error)) { txn_memtable_table_tests_raw_pad_0(frame); @@ -9071,6 +9659,8 @@ txn_memtable_table_tests_taken_bool_Debug_fmt(const void *value, StdFormatter *f return std_DebugTuple_finish(&tuple); case TXN_MEMTABLE_MAP_TAKEN_RETRY: return std_fmt_write_str(f, "Retry", 5); + case TXN_MEMTABLE_MAP_TAKEN_MOVED: + return std_fmt_write_str(f, "Moved", 5); default: return std_fmt_write_str(f, "Refused", 7); } @@ -9126,7 +9716,8 @@ txn_memtable_table_tests_first_insert_inside(void *captures, HostIoIo *io, TxnMe *result_0 = txn_memtable_table_memtable(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, c->state, &location); *result_1 = txn_memtable_table_memtable(&TXN_MEMTABLE_MAP_FREE_TREE_OPS, c->state, &location); return txn_memtable_map_finish_Bool(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, txn_memtable_testing_writer(7), false, - 0, false, NULL, 0, txn_memtable_table_tests_free_step, NULL, NULL, + roomed, false, + 0, false, NULL, 0, NULL, txn_memtable_table_tests_free_step, NULL, NULL, txn_memtable_table_tests_free_then, NULL, NULL, result_2, result_3, error) && txn_memtable_table_finish(HOST_IO_IO_TAKES_SHORT(io), prepared, c->change, false, roomed, &handle, error); } @@ -11140,7 +11731,7 @@ test_txn_memtable_table_tests_an_abort_takes_back_the_versions_of_the_keys_its_t std_vec_free_Checked(&handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); STD_ASSERT_EQ(3501, 9, len == 2, &len, std_usize_Debug_fmt, &two, std_usize_Debug_fmt); simc_abort(top); - if (!txn_written_aborted(io, &error)) + if (!txn_written_aborted(io, 0, &error)) STD_RESULT_EXPECT_FAILED(3503, 37, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); txn_memtable_table_tests_handles_are(3504, 3497, 45, io, state, (const uint8 *) "a", 1, &kept, 1); @@ -12445,7 +13036,7 @@ test_txn_memtable_table_tests_a_transaction_past_the_lists_cap_leaves_its_unlist txn_memtable_table_tests_touches(3991, 83, io, &at, (const uint8 *) "past", 4); (void) txn_memtable_table_tests_pushes(3992, 43, io, state, "past", writer); simc_abort(aborted); - if (!txn_written_aborted(io, &error)) + if (!txn_written_aborted(io, 0, &error)) STD_RESULT_EXPECT_FAILED(3994, 37, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); counts[0] = txn_memtable_table_tests_count(io, state, "listed"); @@ -13346,7 +13937,7 @@ test_txn_memtable_table_tests_a_prepare_with_no_tuple_slot_leaves_its_unstored_s TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(7), .op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; TxnMemtableVersionHandle handle; - txn_memtable_table_alone_t alone = {&change, &handle}; + txn_memtable_table_alone_t alone = {.change = &change, .result = &handle}; TxnMemtableTableAim aims[1] = {{.owner = host_shared_Held_record(&held), .tree = &TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, .key = {(const uint8 *) "k", 1}, .need = txn_memtable_version_encoded(&change.writer, &op, change.shown, false, 0)}}; TxnMemtableTableReservation tuple = {UINT64CONST(0xA11)}; @@ -13355,7 +13946,7 @@ test_txn_memtable_table_tests_a_prepare_with_no_tuple_slot_leaves_its_unstored_s HostError error; if (!txn_memtable_table_prepared(io, aims, 1, false, tuple, txn_memtable_table_alone_inside, &alone, NULL, - &has_unstored, &unstored, &error)) + &has_unstored, &unstored, NULL, &error)) STD_RESULT_EXPECT_FAILED(__LINE__, 5, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); STD_ASSERT(__LINE__, 5, !has_unstored && unstored.place == UINT64CONST(0x5EED), @@ -13365,4 +13956,146 @@ test_txn_memtable_table_tests_a_prepare_with_no_tuple_slot_leaves_its_unstored_s simc_TestIo_drop(&test_io); } +/* + * A refused try takes nothing: with one free block left, a prepare whose + * tree has no memtable and whose push needs a whole block for its version is + * refused in one allocation, the memtable's first page with its reserve, and + * leaves no memtable, range record, or page behind for a walk to close and + * free. + */ +void +test_txn_memtable_table_tests_a_refused_try_leaves_no_memtable_range_or_page_behind(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held; + HostSharedRecord state; + HostDeltaPageSize block = {.bytes = (64 << 10) - 8, .words = 1}; + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(7), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + txn_memtable_table_alone_t alone = {.change = &change, .result = &handle}; + TxnMemtableTableAim aims[1]; + TxnMemtableTableReservation none = {0}; + TxnMemtableTableReservation unstored = {0}; + TxnMemtableTableRefusal refusal = {0}; + StdVec filler; + HostError error; + size_t ranges_before; + size_t pages_before; + size_t ranges_after; + size_t pages_after; + uint64 location; + bool has_unstored = false; + + simc_size_delta(UINT64CONST(4) << 16); + held = txn_memtable_testing_created(io, 9791); + state = host_shared_Held_record(&held); + aims[0] = (TxnMemtableTableAim) + { + .owner = state, .tree = &TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, .key = + { + (const uint8 *) "k", 1 + }, .need = 40000 + }; + std_vec_new(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + for (;;) + { + StdVec pages; + HostDeltaFull full; + bool allocated; + + if (!host_delta_alloc(io, &block, 1, &allocated, &pages, &full, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!allocated) + break; + std_vec_push_Checked(&filler, &STD_VEC_AT(&pages, HostDeltaOwned, 0), sizeof(HostDeltaOwned), + _Alignof(HostDeltaOwned)); + std_vec_free_Checked(&pages, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + } + STD_ASSERT(__LINE__, 9, filler.len > 0, "!filler.is_empty()"); + filler.len--; + host_delta_Owned_drop(&STD_VEC_AT(&filler, HostDeltaOwned, filler.len)); + simc_unfreed(TXN_MEMTABLE_RANGE_RANGE_AGED_OPS.shape.id, &ranges_before, &pages_before); + if (!txn_memtable_table_prepared(io, aims, 1, false, none, txn_memtable_table_alone_inside, &alone, NULL, + &has_unstored, &unstored, &refusal, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, refusal.refused, "refusal.refused"); + STD_ASSERT(__LINE__, 9, !txn_memtable_table_memtable(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, state, &location), + "the refused try made no memtable"); + simc_unfreed(TXN_MEMTABLE_RANGE_RANGE_AGED_OPS.shape.id, &ranges_after, &pages_after); + STD_ASSERT_EQ_U64(__LINE__, 9, ranges_after, ranges_before); + STD_ASSERT_EQ_U64(__LINE__, 9, pages_after, pages_before); + std_vec_clear(&filler, sizeof(HostDeltaOwned), host_delta_Owned_drop); + std_vec_free_Checked(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + txn_memtable_Op_drop(&op); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + + +/* + * The refused try of two trees (bugs row 291's review F2): a prepare whose + * two trees, the rows and the free space map, both lack a memtable takes + * the delta's pages in one allocation and then every memtable's range and + * memtable records before it makes either, so a take that fails, the + * allocation or any of the four records, the second tree's included, leaves + * neither memtable and no range record or page behind (bugs row 321). + */ +void +test_txn_memtable_table_tests_a_refused_try_of_two_trees_leaves_neither_memtable_behind(void) +{ + for (uint64 after = 0; after < 5; after++) + { + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9792); + HostSharedRecord state = host_shared_Held_record(&held); + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(7), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + txn_memtable_table_alone_t alone = {.change = &change, .result = &handle}; + static const uint8 block[4] = {0}; + TxnMemtableTableAim aims[2] = { + {.owner = state, .tree = &TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, .key = {(const uint8 *) "k", 1}, .need = 64}, + {.owner = state, .tree = &TXN_MEMTABLE_MAP_FREE_TREE_OPS, .key = {block, sizeof(block)}, .need = 64}}; + TxnMemtableTableReservation none = {0}; + TxnMemtableTableReservation unstored = {0}; + TxnMemtableTableRefusal refusal = {0}; + HostError error; + size_t ranges_before; + size_t pages_before; + size_t ranges_after; + size_t pages_after; + uint64 location; + bool has_unstored = false; + bool prepared; + + + simc_unfreed(TXN_MEMTABLE_RANGE_RANGE_AGED_OPS.shape.id, &ranges_before, &pages_before); + simc_refuse_any_take(after); + prepared = txn_memtable_table_prepared(io, aims, 2, false, none, txn_memtable_table_alone_inside, &alone, NULL, + &has_unstored, &unstored, &refusal, &error); + simc_refuse_nothing(); + if (prepared) + STD_ASSERT(__LINE__, 9, refusal.refused, "a try refused at a take is refused, or fails"); + else + host_Error_drop(&error); + STD_ASSERT(__LINE__, 9, !txn_memtable_table_memtable(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, state, &location), + "the refused try made no rows memtable"); + STD_ASSERT(__LINE__, 9, !txn_memtable_table_memtable(&TXN_MEMTABLE_MAP_FREE_TREE_OPS, state, &location), + "the refused try made no free space map memtable"); + simc_unfreed(TXN_MEMTABLE_RANGE_RANGE_AGED_OPS.shape.id, &ranges_after, &pages_after); + STD_ASSERT_EQ_U64(__LINE__, 9, ranges_after, ranges_before); + STD_ASSERT_EQ_U64(__LINE__, 9, pages_after, pages_before); + txn_memtable_Op_drop(&op); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); + } +} + #endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/memtable/testing.c b/src/doltgres/txn/c/memtable/testing.c index 48cac81a1ad98..bea26baa9eb00 100644 --- a/src/doltgres/txn/c/memtable/testing.c +++ b/src/doltgres/txn/c/memtable/testing.c @@ -310,7 +310,8 @@ txn_memtable_testing_pushed_body_0(void *frame_) std_vec_new(&frame->buffer, 1, 1); frame->buffer_live = true; frame->ok = txn_memtable_version_push(&open, &frame->hold, frame->key, - frame->key_len, &change, &frame->has_fresh, &frame->fresh, &frame->pages, + frame->key_len, &change, false, &frame->has_fresh, &frame->fresh, + &frame->pages, &frame->buffer, frame->result, frame->error); txn_memtable_testing_pushed_pad_0(frame); } diff --git a/src/doltgres/txn/c/memtable/tuples.c b/src/doltgres/txn/c/memtable/tuples.c index 79776009ff825..de90dcdcb0e56 100644 --- a/src/doltgres/txn/c/memtable/tuples.c +++ b/src/doltgres/txn/c/memtable/tuples.c @@ -52,6 +52,8 @@ #define TXN_MEMTABLE_TUPLES_USED 1 #define TXN_MEMTABLE_TUPLES_RESERVED 2 #define TXN_MEMTABLE_TUPLES_COUNT 3 +/* TXN_MEMTABLE_TUPLES_DEAD_COUNT counts a table's slots whose tids a spill dropped, until a growth or a compaction leaves them behind. */ +#define TXN_MEMTABLE_TUPLES_DEAD_COUNT 4 /* TXN_MEMTABLE_TUPLES_PER is PER, the slots a slot page holds, two words each. */ #define TXN_MEMTABLE_TUPLES_PER UINT64CONST(512) @@ -75,6 +77,13 @@ /* TXN_MEMTABLE_TUPLES_DEAD is DEAD, the tuple id word a dropped slot holds, which a search passes over. */ #define TXN_MEMTABLE_TUPLES_DEAD PG_UINT64_MAX +/* + * TXN_MEMTABLE_TUPLES_DROPPED is the entry, in a table's list of byte pages, + * of a byte page a compaction retired, which keeps every other page's index, + * so no key's place moves; no page's name is zero. + */ +#define TXN_MEMTABLE_TUPLES_DROPPED 0 + StaticAssertDecl(HOST_DELTA_PAGE_LARGEST <= UINT64CONST(1) << TXN_MEMTABLE_TUPLES_OFFSET && HOST_DELTA_PAGE_LARGEST <= UINT64CONST(1) << TXN_MEMTABLE_TUPLES_LENGTH && HOST_DELTA_PAGE_SLOTS <= UINT64CONST(1) << (64 - TXN_MEMTABLE_TUPLES_OFFSET - TXN_MEMTABLE_TUPLES_LENGTH), @@ -87,9 +96,9 @@ txn_memtable_tuples_Tuples_Aged_count(const HostSharedLimits *limits) return limits->relation_locks; } -/* TXN_MEMTABLE_TUPLES_TUPLES_AGED_OPS is Tuples' Shape and Aged: a storage's tuple id table, its lock, four atomics, and its slot and byte pages. */ +/* TXN_MEMTABLE_TUPLES_TUPLES_AGED_OPS is Tuples' Shape and Aged: a storage's tuple id table, its lock, five atomics, and its slot and byte pages. */ const HostSharedAgedOps TXN_MEMTABLE_TUPLES_TUPLES_AGED_OPS = { - .shape = {.id = UINT64CONST(0x74786e000013), .locks = 1, .atomics = 4, .lists = 2}, + .shape = {.id = UINT64CONST(0x74786e000013), .locks = 1, .atomics = 5, .lists = 2}, .name = {"tuple id table", 14}, .count = txn_memtable_tuples_Tuples_Aged_count, }; @@ -337,7 +346,8 @@ txn_memtable_tuples_Retiring_retire(const TxnMemtableTuplesRetiring *self, HostI * txn_memtable_tuples_closed closes an open table under its exclusive lock, * both lists and then its mark, and gathers its retirement into a live * Retiring: the record, its slot pages, its lists' own pages, and, for a - * table that leaves no next one, its byte pages. + * table that leaves no next one, its byte pages but those a compaction + * retired. */ static bool txn_memtable_tuples_closed(HostSharedGuard *guard, const HostDeltaHold *hold, uint64 location, bool bytes, @@ -394,7 +404,8 @@ txn_memtable_tuples_closed(HostSharedGuard *guard, const HostDeltaHold *hold, ui if (!host_delta_list_ListView_get(&byte_pages, hold, index, &name, error)) return false; - std_vec_push_Checked(&result->names, &name, sizeof(name), _Alignof(uint64)); + if (name != TXN_MEMTABLE_TUPLES_DROPPED) + std_vec_push_Checked(&result->names, &name, sizeof(name), _Alignof(uint64)); } if (!txn_memtable_table_closed_names(&result->names, &slots, hold, error) || !txn_memtable_table_closed_names(&result->names, &keys, hold, error)) @@ -1004,6 +1015,12 @@ typedef struct txn_memtable_tuples_reserve_frame_0_t const uint8 *key; size_t key_len; TxnMemtableTableReservation *result; + + /* + * refused, for a carry's reserve, answers whether the delta's budget + * refused it, with its refusal. + */ + TxnMemtableTableRefusal *refused; HostError *error; bool ok; bool has_want; @@ -1019,9 +1036,12 @@ typedef struct txn_memtable_tuples_reserve_frame_0_t bool guard_live; TxnMemtableTuplesRetiring leaving; bool leaving_live; + /* retired is a Vec of the byte pages a compaction retired. */ + StdVec retired; + bool retired_live; } txn_memtable_tuples_reserve_frame_0_t; -/* txn_memtable_tuples_reserve_pad_0 is reserve's pad: the guard, the retirement, the hold, the try, and its sizes, while live. */ +/* txn_memtable_tuples_reserve_pad_0 is reserve's pad: the guard, the retirement, the retired byte pages, the hold, the try, and its sizes, while live. */ static void txn_memtable_tuples_reserve_pad_0(void *frame_) { @@ -1037,6 +1057,11 @@ txn_memtable_tuples_reserve_pad_0(void *frame_) frame->leaving_live = false; std_vec_free_Checked(&frame->leaving.names, sizeof(uint64), _Alignof(uint64)); } + if (frame->retired_live) + { + frame->retired_live = false; + std_vec_free_Checked(&frame->retired, sizeof(uint64), _Alignof(uint64)); + } if (frame->hold_live) { frame->hold_live = false; @@ -1091,6 +1116,293 @@ txn_memtable_tuples_taken(txn_memtable_tuples_reserve_frame_0_t *frame, bool *ok return true; } +/* + * TxnMemtableTuplesLive is a live slot a compaction stores again: its tid, + * its place, its slot's place word, and the words of the slot it moves to. + */ +typedef struct TxnMemtableTuplesLive +{ + uint64 tid; + uint64 place; + HostDeltaWord place_word; + HostDeltaWord to_tid; + HostDeltaWord to_place; +} TxnMemtableTuplesLive; + +/* txn_memtable_tuples_place_cmp orders live slots by their places, which are distinct. */ +static int +txn_memtable_tuples_place_cmp(const void *a, const void *b) +{ + uint64 x = ((const TxnMemtableTuplesLive *) a)->place; + uint64 y = ((const TxnMemtableTuplesLive *) b)->place; + + return x < y ? -1 : x > y; +} + +/* + * txn_memtable_tuples_moved moves the live keys of a byte page but the open + * one to the open page when they fit its room, storing each slot's new place + * as soon as its key is appended, marks the page DROPPED in the list of + * byte pages, and adds it to the retired pages, which have room for it; live + * holds the page's slots, from its first to the one before end. A failure + * part way leaves every slot naming a place that holds its key. + */ +static bool +txn_memtable_tuples_moved(HostSharedGuard *guard, const HostDeltaHold *hold, uint64 index, uint64 newest, + const HostDeltaPageRef *open, TxnMemtableTuplesLive *live, size_t len, StdVec *retired, + HostError *error) +{ + uint64 mask = (UINT64CONST(1) << TXN_MEMTABLE_TUPLES_LENGTH) - 1; + uint64 bytes = 0; + uint64 name; + HostDeltaPageRef page; + HostDeltaListListMut keys; + + keys = host_delta_list_GuardExclusive_list_mut(guard, &TXN_MEMTABLE_TUPLES_TUPLES_AGED_OPS.shape, + TXN_MEMTABLE_TUPLES_LOCK, TXN_MEMTABLE_TUPLES_KEYS); + { + HostDeltaListListView view = host_delta_list_ListMut_view(&keys); + + if (!host_delta_list_ListView_get(&view, hold, index, &name, error)) + return false; + } + for (size_t k = 0; k < len; k++) + bytes += live[k].place & mask; + if (name == TXN_MEMTABLE_TUPLES_DROPPED || bytes > host_delta_PageRef_room(open)) + return true; + if (!host_delta_Hold_page(hold, host_delta_page_Page_named(name), &page, error)) + return false; + for (size_t k = 0; k < len; k++) + { + uint64 length = live[k].place & mask; + uint64 offset = live[k].place >> TXN_MEMTABLE_TUPLES_LENGTH & ((UINT64CONST(1) << TXN_MEMTABLE_TUPLES_OFFSET) - 1); + HostDeltaLent key; + StdBytes held; + uint32 at; + bool ok; + + if (!host_delta_PageRef_bytes(&page, offset, length, &key, error)) + return false; + held = host_delta_Lent_Deref_deref(&key); + ok = host_delta_PageRef_append(open, &TXN_MEMTABLE_TUPLES_TUPLES_AGED_OPS.shape, TXN_MEMTABLE_TUPLES_LOCK, guard, + held.ptr, held.len, &at, error); + host_delta_Lent_drop(&key); + if (!ok) + return false; + live[k].place = txn_memtable_tuples_placed(newest, at, (size_t) length); + host_delta_Word_store(&live[k].place_word, live[k].place); + } + if (!host_delta_list_ListMut_set(&keys, hold, index, TXN_MEMTABLE_TUPLES_DROPPED, error)) + return false; + std_vec_push_Checked(retired, &name, sizeof(name), _Alignof(uint64)); + return true; +} + +/* + * txn_memtable_tuples_compacted_frame_t is compacted's state across its pad: + * the slot pages, the live slots, and the tid words of every used slot, each + * while live. + */ +typedef struct txn_memtable_tuples_compacted_frame_t +{ + HostSharedGuard *guard; + const HostDeltaHold *hold; + StdVec *retired; + HostError *error; + bool ok; + StdVec pages; + bool pages_live; + StdVec live; + bool live_live; + StdVec cleared; + bool cleared_live; + StdVec taken; + bool taken_live; +} txn_memtable_tuples_compacted_frame_t; + +/* txn_memtable_tuples_compacted_reserved reserves room for more elements in one of compacted's vectors, answering false on running out of memory. */ +static bool +txn_memtable_tuples_compacted_reserved(txn_memtable_tuples_compacted_frame_t *frame, StdVec *vec, size_t more, + size_t size, size_t align) +{ + if (std_vec_try_reserve(vec, more, size, align)) + return true; + *frame->error = host_Error_out_of_memory(); + return false; +} + +/* + * txn_memtable_tuples_compacted_body reads every slot page and gathers the + * live slots and every used slot's tid word, moves the byte pages' keys, + * plans each live slot's new slot and takes its words, and only then zeroes + * the used slots and stores the live ones again, so every step that can + * fail runs before the first slot is zeroed. + */ +static void +txn_memtable_tuples_compacted_body(void *frame_) +{ + txn_memtable_tuples_compacted_frame_t *frame = frame_; + HostSharedGuard *guard = frame->guard; + const HostDeltaHold *hold = frame->hold; + HostSharedRecord record = host_shared_Guard_record(guard); + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + uint64 count = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_COUNT, load); + uint64 held = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_USED, load) - + host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_DEAD_COUNT, load); + uint64 reserved = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_RESERVED, load); + HostDeltaListListView slots = txn_memtable_tuples_view(guard, TXN_MEMTABLE_TUPLES_SLOTS); + uint64 len = host_delta_list_ListView_len(&slots); + HostSharedOp exchange = {.tag = HOST_SHARED_OP_EXCHANGE}; + TxnMemtableTuplesLive *live; + + std_vec_new(&frame->pages, sizeof(HostDeltaPageRef), _Alignof(HostDeltaPageRef)); + frame->pages_live = true; + std_vec_new(&frame->live, sizeof(TxnMemtableTuplesLive), _Alignof(TxnMemtableTuplesLive)); + frame->live_live = true; + std_vec_new(&frame->cleared, sizeof(HostDeltaWord), _Alignof(HostDeltaWord)); + frame->cleared_live = true; + if (!txn_memtable_tuples_compacted_reserved(frame, &frame->pages, (size_t) len, sizeof(HostDeltaPageRef), + _Alignof(HostDeltaPageRef)) || + !txn_memtable_tuples_compacted_reserved(frame, &frame->live, (size_t) held, sizeof(TxnMemtableTuplesLive), + _Alignof(TxnMemtableTuplesLive))) + return; + for (uint64 index = 0; index < len; index++) + { + uint64 name; + HostDeltaPageRef page; + + if (!host_delta_list_ListView_get(&slots, hold, index, &name, frame->error) || + !host_delta_Hold_page(hold, host_delta_page_Page_named(name), &page, frame->error)) + return; + std_vec_push_Checked(&frame->pages, &page, sizeof(page), _Alignof(HostDeltaPageRef)); + for (uint64 at = 0; at < Min(count, TXN_MEMTABLE_TUPLES_PER); at++) + { + HostDeltaWord word; + TxnMemtableTuplesLive slot = {0}; + + if (!host_delta_PageRef_word(&page, 2 * at, &word, frame->error)) + return; + slot.tid = host_delta_Word_load(&word); + if (slot.tid == 0) + continue; + if (!txn_memtable_tuples_compacted_reserved(frame, &frame->cleared, 1, sizeof(HostDeltaWord), + _Alignof(HostDeltaWord))) + return; + std_vec_push_Checked(&frame->cleared, &word, sizeof(word), _Alignof(HostDeltaWord)); + if (slot.tid == TXN_MEMTABLE_TUPLES_DEAD) + continue; + if (!host_delta_PageRef_word(&page, 2 * at + 1, &slot.place_word, frame->error) || + !txn_memtable_tuples_compacted_reserved(frame, &frame->live, 1, sizeof(TxnMemtableTuplesLive), + _Alignof(TxnMemtableTuplesLive))) + return; + slot.place = host_delta_Word_load(&slot.place_word); + std_vec_push_Checked(&frame->live, &slot, sizeof(slot), _Alignof(TxnMemtableTuplesLive)); + } + } + live = frame->live.ptr; + if (reserved == 0 && frame->live.len > 0) + { + HostDeltaListListView keys = txn_memtable_tuples_view(guard, TXN_MEMTABLE_TUPLES_KEYS); + uint64 newest = host_delta_list_ListView_len(&keys) - 1; + size_t at = 0; + uint64 name; + HostDeltaPageRef open; + + qsort(live, frame->live.len, sizeof(TxnMemtableTuplesLive), txn_memtable_tuples_place_cmp); + if (!host_delta_list_ListView_get(&keys, hold, newest, &name, frame->error) || + !host_delta_Hold_page(hold, host_delta_page_Page_named(name), &open, frame->error) || + !txn_memtable_tuples_compacted_reserved(frame, frame->retired, (size_t) newest, sizeof(uint64), + _Alignof(uint64))) + return; + for (uint64 index = 0; index < newest; index++) + { + size_t from = at; + + while (at < frame->live.len && + live[at].place >> (TXN_MEMTABLE_TUPLES_OFFSET + TXN_MEMTABLE_TUPLES_LENGTH) == index) + at++; + if (!txn_memtable_tuples_moved(guard, hold, index, newest, &open, &live[from], at - from, frame->retired, + frame->error)) + return; + } + } + std_vec_new(&frame->taken, sizeof(bool), _Alignof(bool)); + frame->taken_live = true; + if (!txn_memtable_tuples_compacted_reserved(frame, &frame->taken, (size_t) count, sizeof(bool), _Alignof(bool))) + return; + memset(frame->taken.ptr, 0, (size_t) count); + frame->taken.len = (size_t) count; + for (size_t k = 0; k < frame->live.len; k++) + { + uint64 to = txn_memtable_tuples_slot(live[k].tid - 1, count); + bool *taken = frame->taken.ptr; + const HostDeltaPageRef *page; + + while (taken[to]) + to = (to + 1) % count; + taken[to] = true; + page = &STD_VEC_AT(&frame->pages, HostDeltaPageRef, (size_t) (to / TXN_MEMTABLE_TUPLES_PER)); + if (!txn_memtable_tuples_word(page, to, false, 0, 0, &live[k].to_tid, frame->error) || + !txn_memtable_tuples_word(page, to, true, 0, 0, &live[k].to_place, frame->error)) + return; + } + for (size_t k = 0; k < frame->cleared.len; k++) + host_delta_Word_store(&STD_VEC_AT(&frame->cleared, HostDeltaWord, k), 0); + for (size_t k = 0; k < frame->live.len; k++) + { + host_delta_Word_store(&live[k].to_place, live[k].place); + host_delta_Word_store(&live[k].to_tid, live[k].tid); + } + exchange.u.exchange.f0 = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_USED, load); + exchange.u.exchange.f1 = frame->live.len; + host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_USED, exchange); + exchange.u.exchange.f0 = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_DEAD_COUNT, load); + exchange.u.exchange.f1 = 0; + host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_DEAD_COUNT, exchange); + frame->ok = host_pause(HOST_SHARED_GUARD_PAUSES(guard), "doltgres-tuples-compacting", frame->error); +} + +/* txn_memtable_tuples_compacted_pad frees what compacted holds live. */ +static void +txn_memtable_tuples_compacted_pad(void *frame_) +{ + txn_memtable_tuples_compacted_frame_t *frame = frame_; + + if (frame->taken_live) + std_vec_free_Checked(&frame->taken, sizeof(bool), _Alignof(bool)); + if (frame->cleared_live) + std_vec_free_Checked(&frame->cleared, sizeof(HostDeltaWord), _Alignof(HostDeltaWord)); + if (frame->live_live) + std_vec_free_Checked(&frame->live, sizeof(TxnMemtableTuplesLive), _Alignof(TxnMemtableTuplesLive)); + if (frame->pages_live) + std_vec_free_Checked(&frame->pages, sizeof(HostDeltaPageRef), _Alignof(HostDeltaPageRef)); + frame->taken_live = frame->cleared_live = frame->live_live = frame->pages_live = false; +} + +/* + * txn_memtable_tuples_compacted compacts an open table in place under its + * exclusive lock, once its dropped slots leave its live ones at most half + * its load factor, allocating nothing from the delta: its live slots, + * gathered with fallible allocation, are stored again in its own slot pages, + * the dropped ones left behind and its dropped count reset; and, while no + * reservation holds a place on a byte page, each byte page but the open one + * whose live keys fit the open page's room, none included, has them moved + * there with their places and is retired, its entry in the list of byte + * pages marked DROPPED, so no other page's index moves. It adds the retired + * pages to retired, to retire once the lock is released. Every step that can + * fail runs before the first slot is zeroed, and each key's move stores its + * new place at once, so an error or a panic leaves every tid found. + */ +static bool +txn_memtable_tuples_compacted(HostSharedGuard *guard, const HostDeltaHold *hold, StdVec *retired, HostError *error) +{ + txn_memtable_tuples_compacted_frame_t frame = {.guard = guard, .hold = hold, .retired = retired, .error = error}; + + std_landing(&frame, txn_memtable_tuples_compacted_body, txn_memtable_tuples_compacted_pad); + txn_memtable_tuples_compacted_pad(&frame); + return frame.ok; +} + /* * txn_memtable_tuples_reserve_open is a try under an open table's exclusive * lock: it appends the key to the newest byte page when it fits and the table @@ -1113,6 +1425,9 @@ txn_memtable_tuples_reserve_open(txn_memtable_tuples_reserve_frame_0_t *frame, T HostDeltaPageRef page; HostDeltaPageSize byte; TxnMemtableTuplesWant wanted = {0}; + uint64 used; + uint64 reserved; + uint64 dead; *done = false; if (!host_delta_list_ListView_get(&keys, &frame->hold, newest, &name, frame->error) || @@ -1121,8 +1436,17 @@ txn_memtable_tuples_reserve_open(txn_memtable_tuples_reserve_frame_0_t *frame, T byte = txn_memtable_tuples_bytes(frame->key_len, true, STD_ADD(u64, host_delta_PageRef_filled(&page), host_delta_PageRef_room(&page), 280, 40)); - if (txn_memtable_tuples_full(host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_USED, load), - host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_RESERVED, load), count)) + used = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_USED, load); + reserved = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_RESERVED, load); + dead = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_DEAD_COUNT, load); + if (txn_memtable_tuples_full(used, reserved, count) && dead > 0 && + 2 * (used - dead + reserved + 1) <= count / 4 * 3) + { + std_vec_new(&frame->retired, sizeof(uint64), _Alignof(uint64)); + frame->retired_live = true; + return txn_memtable_tuples_compacted(&frame->guard, &frame->hold, &frame->retired, frame->error); + } + if (txn_memtable_tuples_full(used, reserved, count)) { wanted.tag = TXN_MEMTABLE_TUPLES_WANT_GROW; wanted.count = STD_MUL(u64, 2, count, 282, 37); @@ -1191,8 +1515,16 @@ txn_memtable_tuples_reserve_body_0(void *frame_) } if (!ok) { - if (!txn_memtable_table_waited(frame->io, full, false, frame->error)) + bool answered; + + if (!txn_memtable_table_refused(frame->io, full, frame->refused, &answered, frame->error)) + { + txn_memtable_tuples_reserve_pad_0(frame); + return; + } + if (answered) { + frame->ok = true; txn_memtable_tuples_reserve_pad_0(frame); return; } @@ -1256,6 +1588,17 @@ txn_memtable_tuples_reserve_body_0(void *frame_) txn_memtable_tuples_reserve_pad_0(frame); return; } + if (frame->retired_live) + { + TxnMemtableTableRetiring names = {frame->retired.ptr, frame->retired.len, 0}; + StdIterator pages = {&TXN_MEMTABLE_TABLE_RETIRING_ITERATOR_OPS, &names}; + + if (!host_delta_retire(frame->io, NULL, 0, pages, frame->error)) + { + txn_memtable_tuples_reserve_pad_0(frame); + return; + } + } txn_memtable_tuples_reserve_pad_0(frame); } } @@ -1266,17 +1609,21 @@ txn_memtable_tuples_reserve_body_0(void *frame_) * page, so that the finish only stores the slot. Each try takes what the * last one wanted before its hold, then, under the table's exclusive lock: * makes the storage's first table as its first maker when there is none, - * grows the table first when one more slot would pass its load factor, puts - * a fresh byte page on the table's list when the key doesn't fit the newest, - * and counts the slot reserved. + * compacts the table in place when one more slot would pass its load factor + * while its dropped slots leave its live ones at most half of it, else grows + * it first, puts a fresh byte page on the table's list when the key doesn't + * fit the newest, and counts the slot reserved. A take the delta refuses is + * met as refused says (table's refused). */ bool txn_memtable_tuples_reserve(HostIoIo *io, HostSharedRecord state, const uint8 *key, size_t key_len, - TxnMemtableTableReservation *result, HostError *error) + TxnMemtableTableReservation *result, TxnMemtableTableRefusal *refused, HostError *error) { txn_memtable_tuples_reserve_frame_0_t frame = {.io = io, .state = state, .key = key, .key_len = key_len, - .result = result, .error = error}; + .result = result, .refused = refused, .error = error}; + if (refused != NULL) + refused->refused = false; std_landing(&frame, txn_memtable_tuples_reserve_body_0, txn_memtable_tuples_reserve_pad_0); return frame.ok; } @@ -1384,7 +1731,10 @@ txn_memtable_tuples_dropped_one(txn_memtable_tuples_held_frame_0_t *frame, const return true; if (held == STD_ADD(u64, tid, 1, 441, 37)) { + HostSharedOp add = {.tag = HOST_SHARED_OP_ADD, .u.add = 1}; + host_delta_Word_store(&word, TXN_MEMTABLE_TUPLES_DEAD); + host_shared_Record_atomic(host_shared_Guard_record(&frame->guard), TXN_MEMTABLE_TUPLES_DEAD_COUNT, add); return true; } at = STD_REM(u64, STD_ADD(u64, at, 1, 445, 32), count, 445, 31); @@ -1989,6 +2339,14 @@ typedef struct txn_memtable_tuples_tests_counts_t uint64 used; uint64 reserved; uint64 pages; + + /* + * slot_page is the name of the table's last slot page, byte_page of its + * first byte page left, and open_page of its newest. + */ + uint64 slot_page; + uint64 byte_page; + uint64 open_page; } txn_memtable_tuples_tests_counts_t; /* txn_memtable_tuples_tests_counts_frame_0_t is counts' state across its pad: the hold and the guard, while live. */ @@ -2056,12 +2414,36 @@ txn_memtable_tuples_tests_counts_body_0(void *frame_) frame->result->used = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_USED, load); frame->result->reserved = host_shared_Record_atomic(record, TXN_MEMTABLE_TUPLES_RESERVED, load); keys = txn_memtable_tuples_view(&frame->guard, TXN_MEMTABLE_TUPLES_KEYS); - frame->result->pages = host_delta_list_ListView_len(&keys); + frame->result->pages = 0; + frame->result->byte_page = 0; + { + HostDeltaListListView slots = txn_memtable_tuples_view(&frame->guard, TXN_MEMTABLE_TUPLES_SLOTS); + + if (!host_delta_list_ListView_get(&slots, &frame->hold, host_delta_list_ListView_len(&slots) - 1, + &frame->result->slot_page, frame->error)) + { + txn_memtable_tuples_tests_counts_pad_0(frame); + return; + } + } + for (uint64 index = 0, len = host_delta_list_ListView_len(&keys); index < len; index++) + { + uint64 name; + + if (!host_delta_list_ListView_get(&keys, &frame->hold, index, &name, frame->error)) + { + txn_memtable_tuples_tests_counts_pad_0(frame); + return; + } + if (name != TXN_MEMTABLE_TUPLES_DROPPED && frame->result->pages++ == 0) + frame->result->byte_page = name; + frame->result->open_page = name; + } frame->ok = true; txn_memtable_tuples_tests_counts_pad_0(frame); } -/* txn_memtable_tuples_tests_counts returns a storage's table's count of slots, its counts of used and reserved slots, and its count of byte pages. */ +/* txn_memtable_tuples_tests_counts returns a storage's table's count of slots, its counts of used and reserved slots, and its count of byte pages but those a compaction retired. */ static bool txn_memtable_tuples_tests_counts(HostIoIo *io, HostSharedRecord state, txn_memtable_tuples_tests_counts_t *result, HostError *error) @@ -2591,4 +2973,315 @@ test_txn_memtable_tuples_tests_a_tables_first_pages_are_the_smallest_that_hold_t simc_TestIo_drop(&test_io); } +/* txn_memtable_tuples_tests_sized returns the bytes of a storage's table's slot pages and byte pages, as sized does with its unwraps. */ +static uint64 +txn_memtable_tuples_tests_sized(HostIoIo *io, HostSharedRecord state) +{ + HostSharedPast word = txn_memtable_tuples_word_of(state); + HostDeltaHold hold; + HostSharedRecord record; + HostSharedGuard guard; + HostDeltaListListView lists[2]; + HostError error; + uint64 location; + uint64 bytes = 0; + + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 1, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!host_shared_Past_load(&word, &location)) + STD_OPTION_UNWRAP_FAILED(__LINE__, 1); + if (!host_delta_Hold_record(&hold, &TXN_MEMTABLE_TUPLES_TUPLES_AGED_OPS, location, &record, &error) || + !host_shared_Record_lock_shared(record, HOST_IO_IO_TAKES_SHORT(io), TXN_MEMTABLE_TUPLES_LOCK, &guard, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 1, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + lists[0] = txn_memtable_tuples_view(&guard, TXN_MEMTABLE_TUPLES_SLOTS); + lists[1] = txn_memtable_tuples_view(&guard, TXN_MEMTABLE_TUPLES_KEYS); + for (size_t list = 0; list < lengthof(lists); list++) + { + for (uint64 index = 0, len = host_delta_list_ListView_len(&lists[list]); index < len; index++) + { + uint64 name; + HostDeltaPageRef page; + + if (!host_delta_list_ListView_get(&lists[list], &hold, index, &name, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 1, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (list == 1 && name == TXN_MEMTABLE_TUPLES_DROPPED) + continue; + if (!host_delta_Hold_page(&hold, host_delta_page_Page_named(name), &page, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 1, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + bytes += list == 0 ? 8 * host_delta_PageRef_words(&page) : + host_delta_PageRef_filled(&page) + host_delta_PageRef_room(&page); + } + } + host_shared_Guard_drop(&guard); + host_delta_Hold_drop(&hold); + return bytes; +} + +/* + * txn_memtable_tuples_tests_filled inserts the rows of tids 1 to inserted_to, + * drops the slots of tids 1 to dropped_to as a spill drops them, and inserts + * the rows of the tids after inserted_to to last one at a time, the last of + * them past the table's load factor. + */ +static void +txn_memtable_tuples_tests_filled(HostIoIo *io, HostSharedRecord state, uint64 inserted_to, uint64 dropped_to, + uint64 last) +{ + uint64 tids[TXN_MEMTABLE_TUPLES_PER]; + HostError error; + + for (uint64 tid = 1; tid <= inserted_to; tid++) + txn_memtable_tuples_tests_inserts(__LINE__, 1, io, state, tid); + for (uint64 tid = 1; tid <= dropped_to; tid++) + tids[tid - 1] = tid; + if (!txn_memtable_tuples_dropped(io, state, tids, (size_t) dropped_to, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 1, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (uint64 tid = inserted_to + 1; tid <= last; tid++) + txn_memtable_tuples_tests_inserts(__LINE__, 1, io, state, tid); +} + +/* + * The compaction's case: a table whose most slots a spill dropped, past its + * load factor at its next insert, is compacted in place: its live slots + * stored again in its own slot pages, the dropped ones left behind, and + * every byte page but the open one retired, the live keys moved into the + * open page's room, so every live tid is found, every dropped one isn't, and + * the table holds fewer bytes than it held. + */ +void +test_txn_memtable_tuples_tests_a_table_of_mostly_dropped_tids_is_compacted_at_its_load_factor(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9361); + HostSharedRecord state = host_shared_Held_record(&held); + uint64 before; + uint64 after; + + txn_memtable_tuples_tests_filled(io, state, 300, 290, TXN_MEMTABLE_TUPLES_PER / 4 * 3); + before = txn_memtable_tuples_tests_sized(io, state); + txn_memtable_tuples_tests_counts_are(__LINE__, 1, __LINE__, 1, io, state, TXN_MEMTABLE_TUPLES_PER, + TXN_MEMTABLE_TUPLES_PER / 4 * 3, 0, 6); + txn_memtable_tuples_tests_inserts(__LINE__, 1, io, state, 1000); + txn_memtable_tuples_tests_counts_are(__LINE__, 1, __LINE__, 1, io, state, TXN_MEMTABLE_TUPLES_PER, + TXN_MEMTABLE_TUPLES_PER / 4 * 3 - 290 + 1, 0, 1); + after = txn_memtable_tuples_tests_sized(io, state); + STD_ASSERT(__LINE__, 1, after < before, "the compaction retired the byte pages of the dropped keys"); + for (uint64 tid = 291; tid <= TXN_MEMTABLE_TUPLES_PER / 4 * 3; tid++) + txn_memtable_tuples_tests_got_key(__LINE__, 1, __LINE__, 1, io, state, tid); + txn_memtable_tuples_tests_got_key(__LINE__, 1, __LINE__, 1, io, state, 1000); + for (uint64 tid = 1; tid <= 290; tid++) + txn_memtable_tuples_tests_got_is(__LINE__, 1, __LINE__, 1, io, state, tid, NULL, false); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* txn_memtable_tuples_tests_compactions is the compaction cases' writers: each inserts the row of the tid its captures hold, after its database. */ +static bool +txn_memtable_tuples_tests_compactions(void *captures, HostIoIo *io, HostError *error) +{ + const uint64 *given = captures; + + return txn_memtable_tuples_tests_inserter((uint32) given[0], given[1], io, error); +} + +/* + * The compaction's reservation case: a compaction while another prepare's + * reservation holds a place on a byte page moves no byte page: it stores the + * slots again alone, leaving the dropped ones behind and every byte page + * where it was, so the outstanding reservation's finish stores its key's + * place as it was. + */ +void +test_txn_memtable_tuples_tests_a_compaction_with_a_reservation_outstanding_moves_no_byte_page(void) +{ + static const char *const names[2] = {"reserving", "compacting"}; + static const uint64 given[2][2] = {{9362, 2000}, {9362, 1000}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9362); + HostSharedRecord state = host_shared_Held_record(&held); + SimcRun run; + SimcStep steps[2] = {{"compacting", 10, PG_UINT64_MAX}, {"reserving", 9, PG_UINT64_MAX}}; + SimcReport report; + + txn_memtable_tuples_tests_filled(io, state, 300, 290, TXN_MEMTABLE_TUPLES_PER / 4 * 3 - 1); + run = simc_Run_new(171); + for (size_t k = 0; k < lengthof(names); k++) + { + SimcActor actor = {txn_memtable_tuples_tests_compactions, (void *) given[k]}; + + simc_Run_actor(&run, names[k], strlen(names[k]), actor); + } + simc_Run_until(&run, "reserving", 9, "doltgres-memtable-prepared"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_memtable_tuples_tests_endings_are(__LINE__, 1, __LINE__, 1, &report); + txn_memtable_tuples_tests_counts_are(__LINE__, 1, __LINE__, 1, io, state, TXN_MEMTABLE_TUPLES_PER, + TXN_MEMTABLE_TUPLES_PER / 4 * 3 - 290 + 1, 0, 6); + for (uint64 tid = 291; tid < TXN_MEMTABLE_TUPLES_PER / 4 * 3; tid++) + txn_memtable_tuples_tests_got_key(__LINE__, 1, __LINE__, 1, io, state, tid); + txn_memtable_tuples_tests_got_key(__LINE__, 1, __LINE__, 1, io, state, 1000); + txn_memtable_tuples_tests_got_key(__LINE__, 1, __LINE__, 1, io, state, 2000); + simc_Report_drop(&report); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* + * The compaction's lock case: a writer that reaches the table while a + * compaction, paused before it releases the table's exclusive lock, holds it + * waits on the lock and reserves in the compacted table once it ends, so + * both writers' tids and every live tid before them are found. + */ +void +test_txn_memtable_tuples_tests_a_writer_waits_on_a_compactions_exclusive_lock(void) +{ + static const char *const names[2] = {"compacting", "writer"}; + static const uint64 given[2][2] = {{9363, 1000}, {9363, 2000}}; + static const uint64 found[3] = {1000, 2000, 291}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9363); + HostSharedRecord state = host_shared_Held_record(&held); + SimcRun run; + SimcStep steps[2] = {{"writer", 6, PG_UINT64_MAX}, {"compacting", 10, PG_UINT64_MAX}}; + SimcReport report; + txn_memtable_tuples_tests_counts_t counts; + uint64 used = TXN_MEMTABLE_TUPLES_PER / 4 * 3 - 290 + 2; + + txn_memtable_tuples_tests_filled(io, state, 300, 290, TXN_MEMTABLE_TUPLES_PER / 4 * 3); + run = simc_Run_new(173); + for (size_t k = 0; k < lengthof(names); k++) + { + SimcActor actor = {txn_memtable_tuples_tests_compactions, (void *) given[k]}; + + simc_Run_actor(&run, names[k], strlen(names[k]), actor); + } + simc_Run_until(&run, "compacting", 10, "doltgres-tuples-compacting"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_memtable_tuples_tests_endings_are(__LINE__, 1, __LINE__, 1, &report); + for (size_t k = 0; k < lengthof(found); k++) + txn_memtable_tuples_tests_got_key(__LINE__, 1, __LINE__, 1, io, state, found[k]); + counts = txn_memtable_tuples_tests_counted(__LINE__, 1, io, state); + STD_ASSERT_EQ(__LINE__, 1, counts.used == used, &counts.used, std_u64_Debug_fmt, &used, std_u64_Debug_fmt); + simc_Report_drop(&report); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + + +/* TxnMemtableTuplesTestsRefusal is a compaction case's refusal: an allocation, the last slot page's open, the first byte page's, or the open byte page's. */ +typedef enum TxnMemtableTuplesTestsRefusal +{ + TXN_MEMTABLE_TUPLES_TESTS_ALLOCATION, + TXN_MEMTABLE_TUPLES_TESTS_SLOT_PAGE, + TXN_MEMTABLE_TUPLES_TESTS_BYTE_PAGE, + TXN_MEMTABLE_TUPLES_TESTS_OPEN_PAGE, +} TxnMemtableTuplesTestsRefusal; + +/* txn_memtable_tuples_tests_insert_t is a refused compaction's insert's captures: the io, the state, and whether it finished. */ +typedef struct txn_memtable_tuples_tests_insert_t +{ + HostIoIo *io; + HostSharedRecord state; + bool inserted; +} txn_memtable_tuples_tests_insert_t; + +/* txn_memtable_tuples_tests_insert_body inserts tid 1000's row, an error answering it unfinished. */ +static void +txn_memtable_tuples_tests_insert_body(void *captures) +{ + txn_memtable_tuples_tests_insert_t *insert = captures; + HostError error; + + insert->inserted = txn_memtable_tuples_tests_inserted(insert->io, insert->state, 1000, &error); + if (!insert->inserted) + host_Error_drop(&error); +} + +/* + * txn_memtable_tuples_tests_refused fills a table to its compaction, makes + * the compacting insert's step of a refusal fail after a count of them, an + * allocation's failure where it can't be refused panicking, and checks that + * every live tid's key is still found, the insert's tid's too when it + * finished; it answers whether the insert failed or panicked. + */ +static bool +txn_memtable_tuples_tests_refused(uint32 database, TxnMemtableTuplesTestsRefusal refusal, uint64 after) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, database); + HostSharedRecord state = host_shared_Held_record(&held); + txn_memtable_tuples_tests_counts_t counts; + txn_memtable_tuples_tests_insert_t insert = {.io = io, .state = state}; + StdCaught caught; + + txn_memtable_tuples_tests_filled(io, state, 300, 290, TXN_MEMTABLE_TUPLES_PER / 4 * 3); + counts = txn_memtable_tuples_tests_counted(__LINE__, 1, io, state); + switch (refusal) + { + case TXN_MEMTABLE_TUPLES_TESTS_ALLOCATION: + simc_fail_allocation(after + 1); + break; + + case TXN_MEMTABLE_TUPLES_TESTS_SLOT_PAGE: + simc_refuse_open(host_delta_page_Page_named(counts.slot_page), after); + break; + case TXN_MEMTABLE_TUPLES_TESTS_BYTE_PAGE: + simc_refuse_open(host_delta_page_Page_named(counts.byte_page), after); + break; + case TXN_MEMTABLE_TUPLES_TESTS_OPEN_PAGE: + simc_refuse_open(host_delta_page_Page_named(counts.open_page), after); + break; + } + if (!std_catch_unwind(txn_memtable_tuples_tests_insert_body, &insert, &caught)) + std_Caught_drop_Checked(&caught); + simc_fail_allocation(0); + simc_refuse_nothing(); + for (uint64 tid = 291; tid <= TXN_MEMTABLE_TUPLES_PER / 4 * 3; tid++) + txn_memtable_tuples_tests_got_key(__LINE__, 1, __LINE__, 1, io, state, tid); + if (insert.inserted) + txn_memtable_tuples_tests_got_key(__LINE__, 1, __LINE__, 1, io, state, 1000); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); + return !insert.inserted; +} + +/* txn_memtable_tuples_tests_refusals runs a refusal's case after each count of its steps up to a bound, and asserts that one of them failed the insert. */ +static void +txn_memtable_tuples_tests_refusals(uint32 database, TxnMemtableTuplesTestsRefusal refusal, uint64 bound) +{ + uint64 failed = 0; + + for (uint64 after = 0; after < bound; after++) + failed += txn_memtable_tuples_tests_refused(database, refusal, after) ? 1 : 0; + STD_ASSERT_EQ_U64(__LINE__, 1, failed > 0 ? refusal + 1 : 0, refusal + 1); +} + +/* + * The compaction's failure case (bugs row 291's review C1): a compacting + * insert whose allocation or page open fails, or panics, at any of its steps + * (the gather's slot pages, the moves' byte pages and the open page, the + * reinsert's plan) leaves every live tid's key found, since every step that + * can fail runs before the compaction zeroes its first slot and each moved + * key's new place is stored at once. + */ +void +test_txn_memtable_tuples_tests_a_compaction_failed_at_any_step_leaves_every_tid_found(void) +{ + txn_memtable_tuples_tests_refusals(9397, TXN_MEMTABLE_TUPLES_TESTS_ALLOCATION, 200); + txn_memtable_tuples_tests_refusals(9398, TXN_MEMTABLE_TUPLES_TESTS_SLOT_PAGE, 4); + txn_memtable_tuples_tests_refusals(9399, TXN_MEMTABLE_TUPLES_TESTS_BYTE_PAGE, 4); + txn_memtable_tuples_tests_refusals(9388, TXN_MEMTABLE_TUPLES_TESTS_OPEN_PAGE, 4); +} + #endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/memtable/version.c b/src/doltgres/txn/c/memtable/version.c index 8b7008fe12f6e..dfc828c86fae3 100644 --- a/src/doltgres/txn/c/memtable/version.c +++ b/src/doltgres/txn/c/memtable/version.c @@ -37,6 +37,12 @@ #include "doltgres/txn/memtable/pages.h" #include "doltgres/txn/memtable/version.h" +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" +#include "doltgres/txn/memtable/testing.h" +#endif + #define RUST_FILE "txn" STD_SEP "src" STD_SEP "memtable" STD_SEP "version.rs" /* @@ -93,6 +99,13 @@ #define TXN_MEMTABLE_VERSION_ENDED UINT64CONST(16) #define TXN_MEMTABLE_VERSION_SHOWN UINT64CONST(8) +/* + * TXN_MEMTABLE_VERSION_COUNTED is the hint word's mark of a version its push + * counted at its writer's commit status entry, which its removal gives back; + * it lives only in the delta, so no run or WAL record holds it. + */ +#define TXN_MEMTABLE_VERSION_COUNTED UINT64CONST(32) + /* TXN_MEMTABLE_VERSION_SPILLED is the bit of a spilled version's handle. */ #define TXN_MEMTABLE_VERSION_SPILLED (UINT64CONST(1) << 63) @@ -1128,6 +1141,19 @@ txn_memtable_version_OpRef_owned(const TxnMemtableVersionOpRef *self) return op; } +#ifdef DOLTGRES_TEST_TXN +/* + * txn_memtable_version_tests_raced is the tests' one-shot seam inside + * updated's read-modify-write: when set, updated runs it once between the + * word's load and its compare-exchange and clears it, as a reader's store + * landing there would. + */ +static void (*txn_memtable_version_tests_raced) (void *captures); + +/* txn_memtable_version_tests_raced_captures is the seam's captures. */ +static void *txn_memtable_version_tests_raced_captures; +#endif + /* * txn_memtable_version_updated changes a version's hint word by a * read-modify-write, a compare-exchange retried until it lands, so that @@ -1141,8 +1167,18 @@ txn_memtable_version_updated(const HostDeltaWord *word, uint64 keep, uint64 set) for (;;) { - uint64 held = host_delta_Word_compare_exchange(word, current, (current & keep) | set); + uint64 held; + +#ifdef DOLTGRES_TEST_TXN + if (txn_memtable_version_tests_raced != NULL) + { + void (*raced) (void *captures) = txn_memtable_version_tests_raced; + txn_memtable_version_tests_raced = NULL; + raced(txn_memtable_version_tests_raced_captures); + } +#endif + held = host_delta_Word_compare_exchange(word, current, (current & keep) | set); if (held == current) return; current = held; @@ -1304,7 +1340,7 @@ txn_memtable_version_Version_ender(const TxnMemtableVersionVersion *self, uint32 void txn_memtable_version_Version_end(TxnMemtableVersionVersion *self, uint32 ender) { - txn_memtable_version_Version_update(self, TXN_MEMTABLE_VERSION_HINT, + txn_memtable_version_Version_update(self, TXN_MEMTABLE_VERSION_HINT | TXN_MEMTABLE_VERSION_COUNTED, TXN_MEMTABLE_VERSION_ENDED | (uint64) ender << TXN_MEMTABLE_VERSION_TOKEN); } @@ -1321,10 +1357,15 @@ txn_memtable_version_Version_stored(const TxnMemtableVersionVersion *self, StdVe std_slice_to_vec_Checked(result, record.f0.ptr, record.f0.len, 1, 1); if (txn_memtable_version_Version_ender(self, &ender)) - { - ((uint8 *) result->ptr)[28] |= TXN_MEMTABLE_VERSION_ENDER; - txn_memtable_version_le(result, ender, 4); - } + txn_memtable_version_ended(result, ender); +} + +/* txn_memtable_version_ended adds to a free space take's encoded record ENDER and the 32-bit xid of the ender of the row its tuple id held, as a run holds it. */ +void +txn_memtable_version_ended(StdVec *record, uint32 ender) +{ + ((uint8 *) record->ptr)[28] |= TXN_MEMTABLE_VERSION_ENDER; + txn_memtable_version_le(record, ender, 4); } /* txn_memtable_version_Version_confirm clears a speculative insert's token, as heap_finish_speculative does. */ @@ -1334,6 +1375,13 @@ txn_memtable_version_Version_confirm(TxnMemtableVersionVersion *self) txn_memtable_version_Version_update(self, (UINT64CONST(1) << TXN_MEMTABLE_VERSION_TOKEN) - 1, 0); } +/* txn_memtable_version_Version_counted reports whether the version's push counted it at its writer's entry. */ +bool +txn_memtable_version_Version_counted(const TxnMemtableVersionVersion *self) +{ + return (txn_memtable_version_Version_word(self) & TXN_MEMTABLE_VERSION_COUNTED) != 0; +} + /* txn_memtable_version_Version_killed reports whether a speculative insert's writer killed the version. */ bool txn_memtable_version_Version_killed(const TxnMemtableVersionVersion *self) @@ -1567,7 +1615,9 @@ txn_memtable_version_show(const TxnMemtableIndexKeyedOps *keyed, const HostShare return true; if (!txn_memtable_version_version(keyed, guard, hold, handle, &version, error)) return false; - txn_memtable_version_Version_update(&version, TXN_MEMTABLE_VERSION_HINT | TXN_MEMTABLE_VERSION_KILLED, + txn_memtable_version_Version_update(&version, + TXN_MEMTABLE_VERSION_HINT | TXN_MEMTABLE_VERSION_KILLED | + TXN_MEMTABLE_VERSION_COUNTED, TXN_MEMTABLE_VERSION_SHOWN | (uint64) combo << TXN_MEMTABLE_VERSION_TOKEN); txn_memtable_version_Version_drop(&version); return true; @@ -1634,15 +1684,34 @@ txn_memtable_version_Chain_Iterator_next(TxnMemtableVersionChain *self, bool *ha * new key's node with the version as its head, are placed together on the * range's newest page or on the fresh page given; every step that can fail * runs first, and the store of the key's head, or of the new node's links, - * publishes it. + * publishes it. A counted version's hint word carries the counted mark. */ bool txn_memtable_version_push(TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, const uint8 *key, - size_t key_len, const TxnMemtableVersionChange *change, bool *has_fresh, + size_t key_len, const TxnMemtableVersionChange *change, bool counted, bool *has_fresh, HostDeltaOwned *fresh, StdVec *spare, StdVec *buffer, TxnMemtableVersionPushed *result, HostError *error) { TxnMemtableIndexFound found; + + return txn_memtable_index_lookup(open, hold, key, key_len, &found, error) && + txn_memtable_version_push_found(open, hold, found, key, key_len, change, counted, has_fresh, fresh, spare, + buffer, result, error); +} + +/* + * txn_memtable_version_push_found is push with the key's lookup in the same + * open already made, its Found taken by value from a lookup its caller made + * under this open's lock, which it still holds, with nothing stored into the + * index since: a Found names links the lock keeps as they were (bugs row 355). + */ +bool +txn_memtable_version_push_found(TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, + TxnMemtableIndexFound found, const uint8 *key, size_t key_len, + const TxnMemtableVersionChange *change, bool counted, bool *has_fresh, + HostDeltaOwned *fresh, StdVec *spare, StdVec *buffer, + TxnMemtableVersionPushed *result, HostError *error) +{ size_t bytes; uint64 words; uint64 older = TXN_MEMTABLE_INDEX_NONE; @@ -1654,8 +1723,6 @@ txn_memtable_version_push(TxnMemtableIndexKeyedOpen *open, const HostDeltaHold * txn_memtable_version_encode(&change->writer, change->op, change->lock, change->shown, change->has_lsn, change->lsn, buffer); - if (!txn_memtable_index_lookup(open, hold, key, key_len, &found, error)) - return false; if (found.tag == TXN_MEMTABLE_INDEX_FOUND_NODE) { bytes = buffer->len; @@ -1679,7 +1746,9 @@ txn_memtable_version_push(TxnMemtableIndexKeyedOpen *open, const HostDeltaHold * handle = txn_memtable_version_Handle_at(room.page, room.first); if (!txn_memtable_pages_Room_append(&room, buffer->ptr, buffer->len, &offset, error) || !txn_memtable_pages_Room_store(&room, 0, txn_memtable_version_linked(offset, older), error) || - !txn_memtable_pages_Room_store(&room, 1, TXN_MEMTABLE_VERSION_HINT_UNKNOWN, error)) + !txn_memtable_pages_Room_store(&room, 1, + TXN_MEMTABLE_VERSION_HINT_UNKNOWN | (counted ? TXN_MEMTABLE_VERSION_COUNTED : 0), + error)) { txn_memtable_pages_Room_drop(&room); return false; @@ -1721,6 +1790,7 @@ typedef struct txn_memtable_version_removed_frame_t size_t handles_len; bool *taken; size_t taken_len; + StdVec *counted; StdVec touched; bool touched_live; bool *result_0; @@ -1740,6 +1810,42 @@ txn_memtable_version_removed_closure_0(const txn_memtable_version_removed_frame_ error); } +/* + * txn_memtable_version_removed_counted appends to removed's counted list the + * top-level xid of a version it takes out whose hint word carries the counted + * mark, read from its record at an offset of its page. + */ +static bool +txn_memtable_version_removed_counted(txn_memtable_version_removed_frame_t *frame, const HostDeltaPageRef *page, + uint64 word, uint64 offset, uint64 bytes) +{ + HostDeltaWord hint; + HostDeltaLent lent; + StdBytes read; + TxnMemtableVersionRecord record; + TxnMemtableVersionWriter writer; + uint64 top; + + if (!host_delta_PageRef_word(page, word + 1, &hint, frame->error)) + return false; + if ((host_delta_Word_load(&hint) & TXN_MEMTABLE_VERSION_COUNTED) == 0) + return true; + if (!host_delta_PageRef_bytes(page, offset, bytes, &lent, frame->error)) + return false; + read = host_delta_Lent_Deref_deref(&lent); + record = txn_memtable_version_Record_of(read.ptr, read.len); + writer = txn_memtable_version_Record_writer(&record); + top = writer.has_top ? writer.top : writer.xid; + host_delta_Lent_drop(&lent); + if (!std_vec_try_reserve(frame->counted, 1, sizeof(uint64), _Alignof(uint64))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(frame->counted, &top, sizeof(uint64), _Alignof(uint64)); + return true; +} + /* * txn_memtable_version_removed_body walks the chain from its head, linking * past each version a handle names and counting its page's live record and @@ -1802,6 +1908,9 @@ txn_memtable_version_removed_body(void *frame_) removed = STD_ADD(u64, removed, bytes, 974, 17); std_index_check(place, frame->taken_len, RUST_FILE, 975, 17); frame->taken[place] = true; + if (!txn_memtable_version_removed_counted(frame, &page, word, host_delta_Word_load(&link) >> + TXN_MEMTABLE_VERSION_OFFSET, bytes)) + return; for (size_t i = 0; i < frame->touched.len && !touched; i++) touched = STD_VEC_AT(&frame->touched, uint64, i) == entry; if (!touched) @@ -1846,19 +1955,20 @@ txn_memtable_version_removed_pad(void *frame_) * the key's head moves to a removed newest version's older, or the newer * version's link takes the removed one's older, and each removed version's * page counts one live record less and its record's bytes dead. It marks - * each handle it took out at its place in taken, and answers whether the - * chain is left empty, the list entries of the pages it took versions from, - * and the bytes of the records it took out. + * each handle it took out at its place in taken, appends to counted the + * top-level xid of each it took out that its push counted, and answers + * whether the chain is left empty, the list entries of the pages it took + * versions from, and the bytes of the records it took out. */ bool txn_memtable_version_removed(const TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, TxnMemtableIndexNode node, const TxnMemtableVersionHandle *handles, size_t handles_len, - bool *taken, size_t taken_len, bool *result_0, StdVec *result_1, uint64 *result_2, - HostError *error) + bool *taken, size_t taken_len, StdVec *counted, bool *result_0, StdVec *result_1, + uint64 *result_2, HostError *error) { txn_memtable_version_removed_frame_t frame = {.open = open, .hold = hold, .node = node, .handles = handles, - .handles_len = handles_len, .taken = taken, .taken_len = taken_len, .result_0 = result_0, - .result_1 = result_1, .result_2 = result_2, .error = error}; + .handles_len = handles_len, .taken = taken, .taken_len = taken_len, .counted = counted, + .result_0 = result_0, .result_1 = result_1, .result_2 = result_2, .error = error}; std_landing(&frame, txn_memtable_version_removed_body, txn_memtable_version_removed_pad); txn_memtable_version_removed_pad(&frame); @@ -2109,3 +2219,768 @@ txn_memtable_version_Pushed_Debug_fmt(const void *value, StdFormatter *f) return txn_memtable_version_variant_Debug_fmt(f, "Wants", &self->u.wants, txn_memtable_pages_Want_Debug_fmt, NULL, NULL); } + +#ifdef DOLTGRES_TEST_TXN +/* TXN_MEMTABLE_VERSION_TESTS_UNWRAP is Result::unwrap's panic text. */ +#define TXN_MEMTABLE_VERSION_TESTS_UNWRAP "called `Result::unwrap()` on an `Err` value" + +/* txn_memtable_version_tests_ok unwraps a host result at a line. */ +static void +txn_memtable_version_tests_ok(bool ok, HostError *error, uint32 line) +{ + if (!ok) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_MEMTABLE_VERSION_TESTS_UNWRAP, error, host_Error_Debug_fmt); +} + +/* txn_memtable_version_tests_read_t is read's (u64, Deleter, Hint): a version's handle, its older version's deleter, and its hint. */ +typedef struct txn_memtable_version_tests_read_t +{ + uint64 handle; + TxnMemtableVersionDeleter deleter; + TxnMemtableVersionHint hint; +} txn_memtable_version_tests_read_t; + +/* txn_memtable_version_tests_opened takes a hold, the Keys record at a location, and its lock shared, then finds a key's node. */ +static void +txn_memtable_version_tests_opened(HostIoIo *io, uint64 keys, const uint8 *key, size_t key_len, HostDeltaHold *hold, + HostSharedGuard *guard, TxnMemtableIndexNode *node, uint32 line) +{ + HostSharedRecord record; + bool some = false; + HostError error; + + txn_memtable_version_tests_ok(host_delta_hold(io, hold, &error) && + host_delta_Hold_record(hold, &TXN_MEMTABLE_TESTING_KEYS_AGED_OPS, keys, &record, + &error) && + host_shared_Record_lock_shared(record, HOST_IO_IO_TAKES_SHORT(io), + TXN_MEMTABLE_TESTING_KEYS_LOCK, guard, &error) && + txn_memtable_index_find(&TXN_MEMTABLE_TESTING_KEYS_KEYED_OPS, guard, hold, key, + key_len, &some, node, &error), &error, line); + STD_ASSERT_MSG(line, 0, some, "no key"); +} + +/* txn_memtable_version_tests_newest returns a found key's newest version, which unwraps at a line. */ +static TxnMemtableVersionVersion +txn_memtable_version_tests_newest(const HostSharedGuard *guard, const HostDeltaHold *hold, TxnMemtableIndexNode node, + uint32 line) +{ + TxnMemtableVersionChain chain; + TxnMemtableVersionVersion version; + bool some = false; + HostError error; + + txn_memtable_version_tests_ok(txn_memtable_version_chain(&TXN_MEMTABLE_TESTING_KEYS_KEYED_OPS, guard, hold, node, + &chain, &error) && + txn_memtable_version_Chain_Iterator_next(&chain, &some, &version, &error), &error, + line); + STD_ASSERT_MSG(line, 0, some, "a key with no version"); + return version; +} + +/* txn_memtable_version_tests_read reads a key's versions newest first, each as its handle, its older version's deleter, and its hint. */ +static StdVec +txn_memtable_version_tests_read(HostIoIo *io, uint64 keys, const uint8 *key, size_t key_len) +{ + HostDeltaHold hold; + HostSharedGuard guard; + TxnMemtableIndexNode node; + TxnMemtableVersionChain chain; + StdVec read; + HostError error; + + txn_memtable_version_tests_opened(io, keys, key, key_len, &hold, &guard, &node, __LINE__); + txn_memtable_version_tests_ok(txn_memtable_version_chain(&TXN_MEMTABLE_TESTING_KEYS_KEYED_OPS, &guard, &hold, node, + &chain, &error), &error, __LINE__); + std_vec_new(&read, sizeof(txn_memtable_version_tests_read_t), _Alignof(txn_memtable_version_tests_read_t)); + for (;;) + { + TxnMemtableVersionVersion version; + txn_memtable_version_tests_read_t one; + bool some; + + txn_memtable_version_tests_ok(txn_memtable_version_Chain_Iterator_next(&chain, &some, &version, &error), + &error, __LINE__); + if (!some) + break; + one.handle = version.handle.f0; + one.deleter = txn_memtable_version_Version_deleter(&version); + one.hint = txn_memtable_version_Version_hint(&version); + txn_memtable_version_Version_drop(&version); + std_vec_push_Checked(&read, &one, sizeof(one), _Alignof(txn_memtable_version_tests_read_t)); + } + host_shared_Guard_drop(&guard); + host_delta_Hold_drop(&hold); + return read; +} + +/* txn_memtable_version_tests_pushed pushes a key's change, answering the version it pushed, which unwraps at a line. */ +static uint64 +txn_memtable_version_tests_pushed(HostIoIo *io, uint64 keys, const uint8 *key, size_t key_len, TxnMemtableOp op, + HostDeltaPageSize size, uint32 line) +{ + TxnMemtableVersionPushed pushed; + HostError error; + + txn_memtable_version_tests_ok(txn_memtable_testing_pushed(io, keys, key, key_len, &op, size, &pushed, &error), + &error, line); + txn_memtable_Op_drop(&op); + STD_ASSERT_MSG(line, 0, pushed.tag == TXN_MEMTABLE_VERSION_PUSHED_VERSION, "explicit panic"); + return pushed.u.version.f0; +} + +/* txn_memtable_version_tests_numbered returns key 1 of the testing keys. */ +static StdVec +txn_memtable_version_tests_one_key(void) +{ + StdVec key; + + txn_memtable_testing_numbered(1, &key); + return key; +} + +/* txn_memtable_version_tests_put returns a put of a test's value with its addresses and value ids, under row 9. */ +static TxnMemtableOp +txn_memtable_version_tests_put(uint8 tag, size_t addresses, size_t values, uint32 heap_bytes) +{ + ChunksTrailerLocator locator = chunks_trailer_Locator_at(UINT64CONST(0xabcdef), UINT64CONST(0x123456789a), + UINT64CONST(0xfedcba)); + static const uint32 ids[] = {11, 0xfffffffe}; + TxnMemtableOp op = {.tag = tag}; + + std_slice_to_vec_Checked(&op.u.put.value, "value", 5, 1, 1); + std_vec_new(&op.u.put.addresses, sizeof(chunks_TupleU32Locator_t), _Alignof(chunks_TupleU32Locator_t)); + for (uint32 k = 0; k < addresses; k++) + { + chunks_TupleU32Locator_t address = {3 + k, locator}; + + std_vec_push_Checked(&op.u.put.addresses, &address, sizeof(address), _Alignof(chunks_TupleU32Locator_t)); + } + std_slice_to_vec_Checked(&op.u.put.values, ids, values, sizeof(uint32), _Alignof(uint32)); + op.u.put.tuple.f0 = 9; + op.u.put.heap_bytes = heap_bytes; + return op; +} + +/* txn_memtable_version_tests_moved returns a successor: another key, "moved", or another table. */ +static TxnMemtableSuccessor +txn_memtable_version_tests_moved(bool other) +{ + TxnMemtableSuccessor successor = {.tag = other ? TXN_MEMTABLE_SUCCESSOR_OTHER_TABLE : TXN_MEMTABLE_SUCCESSOR_KEY}; + + if (!other) + std_slice_to_vec_Checked(&successor.u.key, "moved", 5, 1, 1); + return successor; +} + +/* txn_memtable_version_tests_ops returns every op the record test writes, in the Rust test's order. */ +static StdVec +txn_memtable_version_tests_ops(void) +{ + StdVec ops; + TxnMemtableOp op; + + std_vec_new(&ops, sizeof(TxnMemtableOp), _Alignof(TxnMemtableOp)); +#define TXN_MEMTABLE_VERSION_TESTS_OP(made) \ + do { op = (made); std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); } while (0) + TXN_MEMTABLE_VERSION_TESTS_OP(txn_memtable_version_tests_put(TXN_MEMTABLE_OP_PUT, 2, 2, 44)); + TXN_MEMTABLE_VERSION_TESTS_OP(txn_memtable_version_tests_put(TXN_MEMTABLE_OP_PUT, 1, 0, 32)); + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_DELETE + }; + std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_EMPTIED + }; + op.u.emptied.f0 = 9; + std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); + for (int other = 0; other < 2; other++) + { + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_MOVE + }; + op.u.move = txn_memtable_version_tests_moved(other); + std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); + } + TXN_MEMTABLE_VERSION_TESTS_OP(txn_memtable_version_tests_put(TXN_MEMTABLE_OP_ADD, 2, 2, 44)); + for (int moved = 0; moved < 3; moved++) + { + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_REMOVE + }; + op.u.remove.f0.f0 = 5; + op.u.remove.has_f1 = moved != 0; + if (moved != 0) + op.u.remove.f1 = txn_memtable_version_tests_moved(moved == 2); + std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); + } + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_LOCK + }; + op.u.lock = TXN_MEMTABLE_LOCK_MODE_SHARE; + std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); + TXN_MEMTABLE_VERSION_TESTS_OP(txn_memtable_version_tests_put(TXN_MEMTABLE_OP_ADD_ROW, 2, 2, 44)); + { + TxnMemtableOp put = txn_memtable_version_tests_put(TXN_MEMTABLE_OP_PUT, 2, 2, 44); + + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_UPDATE_ROW + }; + op.u.update_row.f0.f0 = 6; + op.u.update_row.f1 = put.u.put; + std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); + } + for (int moved = 0; moved < 3; moved++) + { + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_DELETE_ROW + }; + op.u.delete_row.f0.f0 = 6; + op.u.delete_row.has_f1 = moved != 0; + if (moved != 0) + op.u.delete_row.f1 = txn_memtable_version_tests_moved(moved == 2); + std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); + } + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_LOCK_ROW + }; + op.u.lock_row.f0.f0 = 6; + op.u.lock_row.f1 = TXN_MEMTABLE_LOCK_MODE_KEY_SHARE; + std_vec_push_Checked(&ops, &op, sizeof(op), _Alignof(TxnMemtableOp)); +#undef TXN_MEMTABLE_VERSION_TESTS_OP + return ops; +} + +/* txn_memtable_version_tests_puts reports whether a put read in place holds what a put was written with. */ +static bool +txn_memtable_version_tests_puts(const TxnMemtableVersionPutRef *read, const TxnMemtablePut *put) +{ + TxnMemtableVersionAddresses addresses = txn_memtable_version_PutRef_addresses(read); + TxnMemtableVersionValues values = txn_memtable_version_PutRef_values(read); + chunks_TupleU32Locator_t address; + uint32 value; + size_t k = 0; + + if (read->value.len != put->value.len || memcmp(read->value.ptr, put->value.ptr, put->value.len) != 0 || + read->tuple.f0 != put->tuple.f0) + return false; + for (; txn_memtable_version_Addresses_Iterator_next(&addresses, &address); k++) + { + const chunks_TupleU32Locator_t *written = &STD_VEC_AT(&put->addresses, chunks_TupleU32Locator_t, k); + + if (k >= put->addresses.len || address.f0 != written->f0 || + !chunks_trailer_Locator_PartialEq_eq(&address.f1, &written->f1)) + return false; + } + if (k != put->addresses.len) + return false; + for (k = 0; txn_memtable_version_Values_Iterator_next(&values, &value); k++) + if (k >= put->values.len || value != STD_VEC_AT(&put->values, uint32, k)) + return false; + return k == put->values.len; +} + +/* txn_memtable_version_tests_one reports whether a successor read in place is the one written. */ +static bool +txn_memtable_version_tests_one(const TxnMemtableVersionSuccessorRef *read, const TxnMemtableSuccessor *written) +{ + if (written->tag == TXN_MEMTABLE_SUCCESSOR_OTHER_TABLE) + return read->tag == TXN_MEMTABLE_VERSION_SUCCESSOR_REF_OTHER_TABLE; + return read->tag == TXN_MEMTABLE_VERSION_SUCCESSOR_REF_KEY && read->u.key.len == written->u.key.len && + memcmp(read->u.key.ptr, written->u.key.ptr, read->u.key.len) == 0; +} + +/* txn_memtable_version_tests_moved_is reports whether an optional successor read in place is the one written. */ +static bool +txn_memtable_version_tests_moved_is(bool has_read, const TxnMemtableVersionSuccessorRef *read, bool has_written, + const TxnMemtableSuccessor *written) +{ + return has_read == has_written && (!has_read || txn_memtable_version_tests_one(read, written)); +} + +/* txn_memtable_version_tests_same reports whether a record's op reads back as the op it was written with. */ +static bool +txn_memtable_version_tests_same(const TxnMemtableVersionOpRef *read, const TxnMemtableOp *op) +{ + switch (op->tag) + { + case TXN_MEMTABLE_OP_PUT: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_PUT && txn_memtable_version_tests_puts(&read->u.put, + &op->u.put); + case TXN_MEMTABLE_OP_ADD: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_ADD && txn_memtable_version_tests_puts(&read->u.add, + &op->u.add); + case TXN_MEMTABLE_OP_ADD_ROW: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_ADD_ROW && + txn_memtable_version_tests_puts(&read->u.add_row, &op->u.add_row); + case TXN_MEMTABLE_OP_UPDATE_ROW: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_UPDATE_ROW && + read->u.update_row.f0.f0 == op->u.update_row.f0.f0 && + txn_memtable_version_tests_puts(&read->u.update_row.f1, &op->u.update_row.f1); + case TXN_MEMTABLE_OP_DELETE: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_DELETE; + case TXN_MEMTABLE_OP_EMPTIED: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_EMPTIED && read->u.emptied.f0 == op->u.emptied.f0; + case TXN_MEMTABLE_OP_MOVE: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_MOVE && txn_memtable_version_tests_one(&read->u.move, + &op->u.move); + case TXN_MEMTABLE_OP_REMOVE: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_REMOVE && read->u.remove.f0.f0 == op->u.remove.f0.f0 && + txn_memtable_version_tests_moved_is(read->u.remove.has_f1, &read->u.remove.f1, op->u.remove.has_f1, + &op->u.remove.f1); + case TXN_MEMTABLE_OP_DELETE_ROW: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_DELETE_ROW && + read->u.delete_row.f0.f0 == op->u.delete_row.f0.f0 && + txn_memtable_version_tests_moved_is(read->u.delete_row.has_f1, &read->u.delete_row.f1, + op->u.delete_row.has_f1, &op->u.delete_row.f1); + case TXN_MEMTABLE_OP_LOCK: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_LOCK && read->u.lock == op->u.lock; + case TXN_MEMTABLE_OP_LOCK_ROW: + return read->tag == TXN_MEMTABLE_VERSION_OP_REF_LOCK_ROW && + read->u.lock_row.f0.f0 == op->u.lock_row.f0.f0 && read->u.lock_row.f1 == op->u.lock_row.f1; + default: + return false; + } +} + +/* + * test_txn_memtable_version_tests_every_ops_record_reads_back_as_written: + * every op's record reads back what it was written with, with and without a + * top-level transaction's id and an LSN (at 0, 1, and the largest), with + * each command it shows, a row, a value, addresses, and value ids, and its + * length is what its header and its count of value ids say. + */ +void +test_txn_memtable_version_tests_every_ops_record_reads_back_as_written(void) +{ + StdVec ops = txn_memtable_version_tests_ops(); + TxnMemtableVersionShown shown[] = {{.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}, + {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMBO, .u.combo = 0xfedcba98}, + {.tag = TXN_MEMTABLE_VERSION_SHOWN_REPLAYED}, + {.tag = TXN_MEMTABLE_VERSION_SHOWN_ENDING, .u.ending = 0x12345678}}; + bool has_lsns[] = {false, true, true, true}; + uint64 lsns[] = {0, 0, 1, PG_UINT64_MAX}; + + for (int top = 0; top < 2; top++) + for (size_t l = 0; l < lengthof(lsns); l++) + for (size_t s = 0; s < lengthof(shown); s++) + for (size_t k = 0; k < ops.len; k++) + { + const TxnMemtableOp *op = &STD_VEC_AT(&ops, TxnMemtableOp, k); + TxnMemtableVersionWriter writer = {.xid = UINT64CONST(0x100000007), .has_top = top == 1, + .top = top == 1 ? UINT64CONST(0x100000002) : 0, .command = 3}; + StdVec bytes; + size_t values = 0; + TxnMemtableVersionRecord record; + TxnMemtableVersionWriter read_writer; + uint64 lsn; + bool has_lsn; + TxnMemtableVersionShown read_shown; + uint32 raw = 3; + uint32 ended = 3; + TxnMemtableVersionOpRef read_op; + HostError error; + + std_vec_new(&bytes, 1, 1); + txn_memtable_version_encode(&writer, op, TXN_MEMTABLE_LOCK_MODE_NO_KEY_UPDATE, shown[s], + has_lsns[l], lsns[l], &bytes); + STD_ASSERT_MSG(__LINE__, 9, bytes.len == txn_memtable_version_encoded(&writer, op, shown[s], + has_lsns[l], lsns[l]), + "the record's length is what encoded says"); + if ((((uint8 *) bytes.ptr)[28] & TXN_MEMTABLE_VERSION_VALUES) != 0) + { + size_t at = txn_memtable_version_counted(((uint8 *) bytes.ptr)[28]); + + values = ((uint8 *) bytes.ptr)[at] | (size_t) ((uint8 *) bytes.ptr)[at + 1] << 8; + } + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_length(bytes.ptr, values) == bytes.len, + "the header's length is the record's"); + record = txn_memtable_version_Record_of(bytes.ptr, bytes.len); + read_writer = txn_memtable_version_Record_writer(&record); + has_lsn = txn_memtable_version_Record_lsn(&record, &lsn); + read_shown = txn_memtable_version_Record_shown(&record); + STD_ASSERT_MSG(__LINE__, 9, read_writer.xid == writer.xid && read_writer.has_top == writer.has_top && + read_writer.top == writer.top && read_writer.command == writer.command && + txn_memtable_version_Record_lock(&record) == TXN_MEMTABLE_LOCK_MODE_NO_KEY_UPDATE && + has_lsn == has_lsns[l] && (!has_lsn || lsn == lsns[l]) && + read_shown.tag == shown[s].tag && + (read_shown.tag != TXN_MEMTABLE_VERSION_SHOWN_COMBO || + read_shown.u.combo == shown[s].u.combo) && + (read_shown.tag != TXN_MEMTABLE_VERSION_SHOWN_ENDING || + read_shown.u.ending == shown[s].u.ending), + "(record.writer(), record.lock(), record.lsn(), record.shown()) read back"); + if (shown[s].tag == TXN_MEMTABLE_VERSION_SHOWN_COMBO) + raw = ended = shown[s].u.combo; + else if (shown[s].tag == TXN_MEMTABLE_VERSION_SHOWN_REPLAYED) + raw = ended = 0; + else if (shown[s].tag == TXN_MEMTABLE_VERSION_SHOWN_ENDING) + ended = shown[s].u.ending; + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_Record_raw(&record) == raw && + txn_memtable_version_Record_ended(&record) == ended, + "(record.raw(), record.ended()) == (raw, ended)"); + read_op = txn_memtable_version_Record_op(&record); + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_tests_same(&read_op, op), "an op read as another"); + txn_memtable_version_tests_ok(txn_memtable_version_stored(bytes.ptr, bytes.len, &record, &error), + &error, __LINE__); + { + StdBytes stored = txn_memtable_version_Record_bytes(&record); + + STD_ASSERT_MSG(__LINE__, 9, stored.len == bytes.len && + memcmp(stored.ptr, bytes.ptr, bytes.len) == 0, + "stored(&bytes).unwrap().bytes() == &bytes[..]"); + } + std_vec_free_Checked(&bytes, 1, 1); + } + for (size_t k = 0; k < ops.len; k++) + txn_memtable_Op_drop(&STD_VEC_AT(&ops, TxnMemtableOp, k)); + std_vec_free_Checked(&ops, sizeof(TxnMemtableOp), _Alignof(TxnMemtableOp)); +} + +/* txn_memtable_version_tests_whole returns a record encode writes of an op, by writer 7 with LSN 9. */ +static StdVec +txn_memtable_version_tests_whole(TxnMemtableOp op) +{ + TxnMemtableVersionWriter writer = {.xid = 7}; + TxnMemtableVersionShown shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + StdVec bytes; + + std_vec_new(&bytes, 1, 1); + txn_memtable_version_encode(&writer, &op, TXN_MEMTABLE_LOCK_MODE_UPDATE, shown, true, 9, &bytes); + txn_memtable_Op_drop(&op); + return bytes; +} + +/* txn_memtable_version_tests_changed returns a copy of a record with one byte changed. */ +static StdVec +txn_memtable_version_tests_changed(const StdVec *bytes, size_t at, uint8 to) +{ + StdVec changed; + + std_slice_to_vec_Checked(&changed, bytes->ptr, bytes->len, 1, 1); + ((uint8 *) changed.ptr)[at] = to; + return changed; +} + +/* txn_memtable_version_tests_spliced inserts bytes into a record at a place. */ +static void +txn_memtable_version_tests_spliced(StdVec *bytes, size_t at, const uint8 *inserted, size_t inserted_len) +{ + for (size_t k = 0; k < inserted_len; k++) + std_vec_insert(bytes, at + k, &inserted[k], 1, 1, RUST_FILE, 0, 0); +} + +/* + * test_txn_memtable_version_tests_a_stored_record_is_refused_unless_encode_writes_it: + * a stored record is one encode writes: a record cut short in its header or + * after it, one with a byte past its end, a tag no op has, a row's op with + * no row, a flag no record has, a lock mode past the last, VALUES with no + * ids, REPLAYED with COMBO, and COMBO with ENDING are each refused as data + * corruption. + */ +void +test_txn_memtable_version_tests_a_stored_record_is_refused_unless_encode_writes_it(void) +{ + static const uint8 count[] = {0, 0}; + static const uint8 combo[] = {1, 0, 0, 0}; + TxnMemtableOp locked = {.tag = TXN_MEMTABLE_OP_LOCK, .u.lock = TXN_MEMTABLE_LOCK_MODE_SHARE}; + StdVec row = txn_memtable_version_tests_whole(txn_memtable_testing_put((const uint8 *) "a value past a row's bytes", + strlen("a value past a row's bytes"))); + StdVec lock = txn_memtable_version_tests_whole(locked); + uint8 flags = ((uint8 *) row.ptr)[28]; + StdVec refused[13]; + TxnMemtableVersionRecord record; + HostError error; + + std_slice_to_vec_Checked(&refused[0], row.ptr, TXN_MEMTABLE_VERSION_HEADER - 1, 1, 1); + std_slice_to_vec_Checked(&refused[1], row.ptr, row.len - 1, 1, 1); + std_slice_to_vec_Checked(&refused[2], row.ptr, row.len, 1, 1); + std_vec_u8_push_Checked(&refused[2], 0); + refused[3] = txn_memtable_version_tests_changed(&row, 26, 13); + refused[4] = txn_memtable_version_tests_changed(&row, 26, 14); + refused[5] = txn_memtable_version_tests_changed(&row, 26, TXN_MEMTABLE_VERSION_LOCK_TAG + 4); + refused[6] = txn_memtable_version_tests_changed(&lock, 26, TXN_MEMTABLE_VERSION_ROW_TAG + 1); + refused[7] = txn_memtable_version_tests_changed(&lock, 26, TXN_MEMTABLE_VERSION_ROW_LOCK_TAG); + refused[8] = txn_memtable_version_tests_changed(&row, 28, flags | 128); + refused[9] = txn_memtable_version_tests_changed(&row, 27, 4); + refused[10] = txn_memtable_version_tests_changed(&row, 28, flags | TXN_MEMTABLE_VERSION_VALUES); + txn_memtable_version_tests_spliced(&refused[10], txn_memtable_version_counted(flags | TXN_MEMTABLE_VERSION_VALUES), + count, sizeof(count)); + refused[11] = txn_memtable_version_tests_changed(&row, 28, flags | TXN_MEMTABLE_VERSION_REPLAYED | + TXN_MEMTABLE_VERSION_COMBO); + txn_memtable_version_tests_spliced(&refused[11], txn_memtable_version_counted(flags | TXN_MEMTABLE_VERSION_COMBO) - 4, + combo, sizeof(combo)); + refused[12] = txn_memtable_version_tests_changed(&row, 28, flags | TXN_MEMTABLE_VERSION_COMBO | + TXN_MEMTABLE_VERSION_ENDING); + txn_memtable_version_tests_spliced(&refused[12], txn_memtable_version_counted(flags | TXN_MEMTABLE_VERSION_COMBO) - 4, + combo, sizeof(combo)); + txn_memtable_version_tests_ok(txn_memtable_version_stored(row.ptr, row.len, &record, &error), &error, __LINE__); + for (size_t at = 0; at < lengthof(refused); at++) + { + bool corrupt = false; + + if (!txn_memtable_version_stored(refused[at].ptr, refused[at].len, &record, &error)) + { + corrupt = memcmp(host_Error_sqlstate(&error), "XX001", 5) == 0; + host_Error_drop(&error); + } + STD_ASSERT_EQ(__LINE__, 13, corrupt, &at, std_usize_Debug_fmt, &at, std_usize_Debug_fmt); + std_vec_free_Checked(&refused[at], 1, 1); + } + std_vec_free_Checked(&lock, 1, 1); + std_vec_free_Checked(&row, 1, 1); +} + +/* + * test_txn_memtable_version_tests_a_link_holds_its_offset_and_older_handle_at_their_limits: + * a link holds its record's offset and its key's older version's handle at + * their limits. + */ +void +test_txn_memtable_version_tests_a_link_holds_its_offset_and_older_handle_at_their_limits(void) +{ + uint64 handle = (HOST_DELTA_PAGE_SLOTS - 1) << TXN_MEMTABLE_VERSION_WORD | + ((UINT64CONST(1) << TXN_MEMTABLE_VERSION_WORD) - 1); + uint64 link = txn_memtable_version_linked((uint32) (TXN_MEMTABLE_PAGES_LARGEST - 1), handle); + TxnMemtableVersionHandle named; + bool has = txn_memtable_version_Handle_named(link & ((UINT64CONST(1) << TXN_MEMTABLE_VERSION_OFFSET) - 1), + &named); + + STD_ASSERT_MSG(__LINE__, 9, link >> TXN_MEMTABLE_VERSION_OFFSET == TXN_MEMTABLE_PAGES_LARGEST - 1 && has && + named.f0 == handle, "(link >> OFFSET, Handle::named(...)) == (LARGEST - 1, Some(handle))"); +} + +/* + * test_txn_memtable_version_tests_a_keys_versions_read_newest_first_with_their_deleters: + * a key's versions read newest first, a new key's first version is its + * node's head, each version shows the next newer one as its deleter, a + * lock's with no command, and a hint is stored and loaded. + */ +void +test_txn_memtable_version_tests_a_keys_versions_read_newest_first_with_their_deleters(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held; + uint64 keys; + StdVec key = txn_memtable_version_tests_one_key(); + TxnMemtableOp lock_op = {.tag = TXN_MEMTABLE_OP_LOCK, .u.lock = TXN_MEMTABLE_LOCK_MODE_UPDATE}; + uint64 first; + uint64 lock; + uint64 last; + StdVec versions; + HostError error; + + txn_memtable_version_tests_ok(txn_memtable_testing_keys(io, &held, &keys, &error), &error, __LINE__); + first = txn_memtable_version_tests_pushed(io, keys, key.ptr, key.len, + txn_memtable_testing_put((const uint8 *) "a", 1), + TXN_MEMTABLE_TESTING_PAGE, __LINE__); + versions = txn_memtable_version_tests_read(io, keys, key.ptr, key.len); + STD_ASSERT_MSG(__LINE__, 9, versions.len == 1 && + STD_VEC_AT(&versions, txn_memtable_version_tests_read_t, 0).handle == first, + "a new key's first version is its node's head"); + std_vec_free_Checked(&versions, sizeof(txn_memtable_version_tests_read_t), + _Alignof(txn_memtable_version_tests_read_t)); + lock = txn_memtable_version_tests_pushed(io, keys, key.ptr, key.len, lock_op, TXN_MEMTABLE_TESTING_PAGE, __LINE__); + last = txn_memtable_version_tests_pushed(io, keys, key.ptr, key.len, + txn_memtable_testing_put((const uint8 *) "b", 1), + TXN_MEMTABLE_TESTING_PAGE, __LINE__); + versions = txn_memtable_version_tests_read(io, keys, key.ptr, key.len); + { + const txn_memtable_version_tests_read_t *v = versions.ptr; + + STD_ASSERT_MSG(__LINE__, 9, versions.len == 3 && v[0].handle == last && v[1].handle == lock && + v[2].handle == first, "the versions read newest first"); + STD_ASSERT_MSG(__LINE__, 9, v[0].deleter.xid == 7 && v[0].deleter.has_command && v[0].deleter.command == 0, + "versions[0].1 == Deleter { xid: 7, command: Some(0) }"); + STD_ASSERT_MSG(__LINE__, 9, v[1].deleter.xid == 7 && !v[1].deleter.has_command, + "versions[1].1 == Deleter { xid: 7, command: None }"); + for (size_t k = 0; k < versions.len; k++) + STD_ASSERT_MSG(__LINE__, 9, v[k].hint == TXN_MEMTABLE_VERSION_HINT_UNKNOWN, "a version's hint is unknown"); + } + std_vec_free_Checked(&versions, sizeof(txn_memtable_version_tests_read_t), + _Alignof(txn_memtable_version_tests_read_t)); + { + HostDeltaHold hold; + HostSharedGuard guard; + TxnMemtableIndexNode node; + TxnMemtableVersionVersion newest; + + txn_memtable_version_tests_opened(io, keys, key.ptr, key.len, &hold, &guard, &node, __LINE__); + newest = txn_memtable_version_tests_newest(&guard, &hold, node, __LINE__); + txn_memtable_version_Version_set_hint(&newest, TXN_MEMTABLE_VERSION_HINT_COMMITTED); + txn_memtable_version_Version_drop(&newest); + host_shared_Guard_drop(&guard); + host_delta_Hold_drop(&hold); + } + versions = txn_memtable_version_tests_read(io, keys, key.ptr, key.len); + { + const txn_memtable_version_tests_read_t *v = versions.ptr; + + STD_ASSERT_MSG(__LINE__, 9, versions.len == 3 && v[0].hint == TXN_MEMTABLE_VERSION_HINT_COMMITTED && + v[1].hint == TXN_MEMTABLE_VERSION_HINT_UNKNOWN && v[2].hint == TXN_MEMTABLE_VERSION_HINT_UNKNOWN, + "hints == [Committed, Unknown, Unknown]"); + } + std_vec_free_Checked(&versions, sizeof(txn_memtable_version_tests_read_t), + _Alignof(txn_memtable_version_tests_read_t)); + std_vec_free_Checked(&key, 1, 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * test_txn_memtable_version_tests_a_version_on_a_fresh_page_is_named_by_its_index_in_the_list: + * a version a push puts on a fresh page is named by that page's index in + * the range's list, read from its handle alone, with no walk of the index. + */ +void +test_txn_memtable_version_tests_a_version_on_a_fresh_page_is_named_by_its_index_in_the_list(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held; + uint64 keys; + HostDeltaPageSize small = {(uint32) (TXN_MEMTABLE_VERSION_HEADER + 40 + 40), + (uint32) TXN_MEMTABLE_PAGES_FIRST + 2 + 2 + 8 + 2}; + uint8 a[40]; + uint8 b[40]; + uint64 first; + uint64 second; + HostError error; + + memset(a, 'a', sizeof(a)); + memset(b, 'b', sizeof(b)); + txn_memtable_version_tests_ok(txn_memtable_testing_keys(io, &held, &keys, &error), &error, __LINE__); + first = txn_memtable_version_tests_pushed(io, keys, a, sizeof(a), txn_memtable_testing_put(NULL, 0), small, + __LINE__); + second = txn_memtable_version_tests_pushed(io, keys, b, sizeof(b), txn_memtable_testing_put(NULL, 0), small, + __LINE__); + STD_ASSERT_MSG(__LINE__, 9, first >> TXN_MEMTABLE_VERSION_WORD == 0 && second >> TXN_MEMTABLE_VERSION_WORD == 1, + "(first.0 >> WORD, second.0 >> WORD) == (0, 1)"); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* txn_memtable_version_tests_hinted is the race test's seam: a reader's hint store landing inside updated's read-modify-write. */ +static void +txn_memtable_version_tests_hinted(void *captures) +{ + txn_memtable_version_Version_set_hint(captures, TXN_MEMTABLE_VERSION_HINT_COMMITTED); +} + +/* + * test_txn_memtable_version_tests_a_hint_and_a_token_clear_racing_both_survive: + * the hint word's read-modify-write (CommPF 7, 17:02), a reader's hint store + * landing between a writer's read of the word and its compare-exchange makes + * the exchange retry, so the hint and the token's clear both survive; and a + * kill keeps the hint and clears the token. + */ +void +test_txn_memtable_version_tests_a_hint_and_a_token_clear_racing_both_survive(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held; + uint64 keys; + StdVec key = txn_memtable_version_tests_one_key(); + HostDeltaHold hold; + HostSharedGuard guard; + TxnMemtableIndexNode node; + TxnMemtableVersionVersion version; + uint32 token; + HostError error; + + txn_memtable_version_tests_ok(txn_memtable_testing_keys(io, &held, &keys, &error), &error, __LINE__); + (void) txn_memtable_version_tests_pushed(io, keys, key.ptr, key.len, + txn_memtable_testing_put((const uint8 *) "a", 1), + TXN_MEMTABLE_TESTING_PAGE, __LINE__); + txn_memtable_version_tests_opened(io, keys, key.ptr, key.len, &hold, &guard, &node, __LINE__); + version = txn_memtable_version_tests_newest(&guard, &hold, node, __LINE__); + STD_ASSERT_MSG(__LINE__, 9, version.has_hint, "version.hint.as_ref().unwrap()"); + host_delta_Word_store(&version.hint, UINT64CONST(7) << TXN_MEMTABLE_VERSION_TOKEN); + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_Version_token(&version, &token) && token == 7, + "version.token() == Some(7)"); + txn_memtable_version_tests_raced = txn_memtable_version_tests_hinted; + txn_memtable_version_tests_raced_captures = &version; + txn_memtable_version_updated(&version.hint, (UINT64CONST(1) << TXN_MEMTABLE_VERSION_TOKEN) - 1, 0); + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_tests_raced == NULL, "the seam ran inside the update"); + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_Version_hint(&version) == TXN_MEMTABLE_VERSION_HINT_COMMITTED && + !txn_memtable_version_Version_token(&version, &token) && !txn_memtable_version_Version_killed(&version), + "(version.hint(), version.token(), version.killed()) == (Hint::Committed, None, false)"); + host_delta_Word_store(&version.hint, UINT64CONST(9) << TXN_MEMTABLE_VERSION_TOKEN | TXN_MEMTABLE_VERSION_HINT_ABORTED); + txn_memtable_version_Version_kill(&version); + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_Version_hint(&version) == TXN_MEMTABLE_VERSION_HINT_ABORTED && + !txn_memtable_version_Version_token(&version, &token) && txn_memtable_version_Version_killed(&version), + "(version.hint(), version.token(), version.killed()) == (Hint::Aborted, None, true)"); + txn_memtable_version_Version_drop(&version); + host_shared_Guard_drop(&guard); + host_delta_Hold_drop(&hold); + std_vec_free_Checked(&key, 1, 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +/* + * test_txn_memtable_version_tests_a_shown_combo_is_the_rows_raw_command_and_no_token: + * a combo command id show stores is the raw command the version's row + * shows, in place of its record's, keeps the hint a reader stored, and reads + * as no token; a version with none shows its record's. + */ +void +test_txn_memtable_version_tests_a_shown_combo_is_the_rows_raw_command_and_no_token(void) +{ + SimcProcess process = simc_process(); + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held; + uint64 keys; + StdVec key = txn_memtable_version_tests_one_key(); + HostDeltaHold hold; + HostSharedGuard guard; + TxnMemtableIndexNode node; + TxnMemtableVersionVersion version; + TxnMemtableVersionRecord record; + uint32 token; + HostError error; + + txn_memtable_version_tests_ok(txn_memtable_testing_keys(io, &held, &keys, &error), &error, __LINE__); + (void) txn_memtable_version_tests_pushed(io, keys, key.ptr, key.len, + txn_memtable_testing_put((const uint8 *) "a", 1), + TXN_MEMTABLE_TESTING_PAGE, __LINE__); + txn_memtable_version_tests_opened(io, keys, key.ptr, key.len, &hold, &guard, &node, __LINE__); + version = txn_memtable_version_tests_newest(&guard, &hold, node, __LINE__); + record = txn_memtable_version_Version_record(&version); + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_Version_raw(&version) == txn_memtable_version_Record_raw(&record), + "version.raw() == version.record().raw()"); + txn_memtable_version_Version_set_hint(&version, TXN_MEMTABLE_VERSION_HINT_COMMITTED); + txn_memtable_version_tests_ok(txn_memtable_version_show(&TXN_MEMTABLE_TESTING_KEYS_KEYED_OPS, &guard, &hold, + version.handle, 0xfedcba98, &error), &error, __LINE__); + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_Version_raw(&version) == 0xfedcba98 && + !txn_memtable_version_Version_token(&version, &token) && + txn_memtable_version_Version_hint(&version) == TXN_MEMTABLE_VERSION_HINT_COMMITTED, + "(version.raw(), version.token(), version.hint()) == (0xfedc_ba98, None, Hint::Committed)"); + txn_memtable_version_tests_ok(txn_memtable_version_show(&TXN_MEMTABLE_TESTING_KEYS_KEYED_OPS, &guard, &hold, + version.handle, 1, &error), &error, __LINE__); + STD_ASSERT_MSG(__LINE__, 9, txn_memtable_version_Version_raw(&version) == 1, "version.raw() == 1"); + txn_memtable_version_Version_drop(&version); + host_shared_Guard_drop(&guard); + host_delta_Hold_drop(&hold); + std_vec_free_Checked(&key, 1, 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); + simc_Process_drop(&process); +} + +#endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/outcome.c b/src/doltgres/txn/c/outcome.c index 18ed2356ae718..2160da28f269b 100644 --- a/src/doltgres/txn/c/outcome.c +++ b/src/doltgres/txn/c/outcome.c @@ -28,6 +28,15 @@ #include "doltgres/txn/outcome.h" #include "doltgres/txn/status.h" +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" +#include "doltgres/txn/memtable/table.h" +#include "doltgres/txn/memtable/testing.h" +#endif + +#define RUST_FILE "txn" STD_SEP "src" STD_SEP "outcome.rs" + /* * txn_outcome_stamped returns whether the transaction of a row stamp's xid is * in progress, committed, or aborted, as outcome answers a version's writer @@ -90,3 +99,91 @@ txn_outcome_outcome(HostIoIo *io, TxnMemtableVersionVersion *version, HostStatus txn_memtable_version_Version_set_hint(version, TXN_MEMTABLE_VERSION_HINT_ABORTED); return true; } + +#ifdef DOLTGRES_TEST_TXN + +/* txn_outcome_tests_written pushes a version of a key into a storage's rows, written by a writer. */ +static void +txn_outcome_tests_written(HostIoIo *io, HostSharedRecord state, const char *key, uint64 xid) +{ + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "row", 3); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + HostError error; + + op.u.put.tuple.f0 = 1; + op.u.put.heap_bytes = 32; + if (!txn_memtable_table_push(io, state, (const uint8 *) key, strlen(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); +} + +/* txn_outcome_tests_outcome returns a version's outcome. */ +static HostStatus +txn_outcome_tests_outcome(HostIoIo *io, TxnMemtableVersionVersion *version) +{ + HostStatus status; + HostError error; + + if (!txn_outcome_outcome(io, version, &status, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return status; +} + +/* + * test_txn_outcome_tests_a_writers_outcome_answers_frozen_with_no_door_and_caches_an_ended_writers: + * a version FrozenTransactionId wrote reads committed with no door asked, so + * its hint stays unset; a running writer's version reads in progress and + * caches nothing; once the writer commits it reads committed and caches it, + * and reads committed from the cache after the commit log keeps no such xid. + */ +void +test_txn_outcome_tests_a_writers_outcome_answers_frozen_with_no_door_and_caches_an_ended_writers(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9330); + HostSharedRecord record = host_shared_Held_record(&held); + HostXid running = simc_begin(); + HostXid later; + HostDeltaHold hold; + StdVec frozen; + StdVec ran; + TxnMemtableVersionVersion *version; + HostError error; + + txn_outcome_tests_written(io, record, "f", 2); + txn_outcome_tests_written(io, record, "r", running.f0); + if (!host_delta_hold(io, &hold, &error) || + !txn_memtable_table_versions(io, &hold, record, (const uint8 *) "f", 1, &frozen, &error) || + !txn_memtable_table_versions(io, &hold, record, (const uint8 *) "r", 1, &ran, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + version = &STD_VEC_AT(&frozen, TxnMemtableVersionVersion, 0); + STD_ASSERT(__LINE__, 9, txn_outcome_tests_outcome(io, version) == HOST_STATUS_COMMITTED && + txn_memtable_version_Version_hint(version) == TXN_MEMTABLE_VERSION_HINT_UNKNOWN, + "a frozen writer's version reads committed and caches nothing"); + version = &STD_VEC_AT(&ran, TxnMemtableVersionVersion, 0); + STD_ASSERT(__LINE__, 9, txn_outcome_tests_outcome(io, version) == HOST_STATUS_IN_PROGRESS && + txn_memtable_version_Version_hint(version) == TXN_MEMTABLE_VERSION_HINT_UNKNOWN, + "a running writer's version reads in progress and caches nothing"); + simc_commit(running); + STD_ASSERT(__LINE__, 9, txn_outcome_tests_outcome(io, version) == HOST_STATUS_COMMITTED && + txn_memtable_version_Version_hint(version) == TXN_MEMTABLE_VERSION_HINT_COMMITTED, + "a committed writer's version reads committed and caches it"); + later = simc_begin(); + simc_truncate_commit_log(later); + STD_ASSERT(__LINE__, 9, txn_outcome_tests_outcome(io, version) == HOST_STATUS_COMMITTED, + "a cached commit reads committed after the commit log keeps no such xid"); + simc_commit(later); + txn_memtable_version_versions_drop(&ran, 0); + txn_memtable_version_versions_drop(&frozen, 0); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +#endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/record.c b/src/doltgres/txn/c/record.c index 0b8ceea20a6cd..57722c65df892 100644 --- a/src/doltgres/txn/c/record.c +++ b/src/doltgres/txn/c/record.c @@ -46,6 +46,10 @@ #include "doltgres/txn/state.h" #include "doltgres/txn/status.h" +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/std/test.h" +#endif + #define RUST_FILE "txn" STD_SEP "src" STD_SEP "record.rs" /* A row change's marks of a counted key's added and removed copies, and of a row added, updated, and deleted. */ @@ -510,6 +514,7 @@ txn_record_read_put(const ChunksRecordRecord *r, HostRow tuple, TxnMemtablePut * const uint8 *raw = chunks_record_Record_bytes(r, 8, &raw_len); size_t value_len; const uint8 *value; + TxnMemtablePut put = {.tuple = tuple}; if (raw == NULL) raw_len = 0; @@ -524,6 +529,7 @@ txn_record_read_put(const ChunksRecordRecord *r, HostRow tuple, TxnMemtablePut * *error = txn_record_corrupt(); return false; } + *result = put; std_vec_with_capacity_Checked(&result->addresses, raw_len / TXN_RECORD_ADDRESS, TXN_RECORD_PLACED); for (size_t at = 0; at < raw_len; at += TXN_RECORD_ADDRESS) { @@ -535,8 +541,6 @@ txn_record_read_put(const ChunksRecordRecord *r, HostRow tuple, TxnMemtablePut * } std_slice_to_vec_Checked(&result->value, value, value_len, 1, 1); std_vec_new(&result->values, TXN_RECORD_ID); - result->tuple = tuple; - result->heap_bytes = 0; return true; } @@ -2550,6 +2554,137 @@ txn_record_StorageEmptied_decode(const uint8 *body, size_t body_len, TxnRecordSt return txn_record_read(&read); } +/* txn_record_unlisted_frame_t is an unlist's encode's state across its pad: its field's bytes and the writer, while live. */ +typedef struct txn_record_unlisted_frame_t +{ + const TxnRecordCarryUnlisted *self; + StdVec *result; + StdVec bytes; + bool bytes_live; + ChunksRecordRecordWriter writer; + bool writer_live; +} txn_record_unlisted_frame_t; + +/* txn_record_unlisted_body writes an unlist's storage and its ends, the first and then each one's distance from the one before. */ +static void +txn_record_unlisted_body(void *frame_) +{ + txn_record_unlisted_frame_t *frame = frame_; + const TxnRecordCarryUnlisted *self = frame->self; + uint64 before = 0; + + frame->writer = chunks_record_RecordWriter_new(); + frame->writer_live = true; + chunks_record_RecordWriter_uint(&frame->writer, 1, self->storage.tablespace); + chunks_record_RecordWriter_uint(&frame->writer, 2, self->storage.database); + chunks_record_RecordWriter_uint(&frame->writer, 3, self->storage.relation); + std_vec_new(&frame->bytes, 1, 1); + frame->bytes_live = true; + for (size_t i = 0; i < self->ends.len; i++) + { + uint64 end = STD_VEC_AT(&self->ends, uint64, i); + + chunks_varint_put_varint(&frame->bytes, end - before); + before = end; + } + chunks_record_RecordWriter_field(&frame->writer, 4, frame->bytes.ptr, frame->bytes.len); + *frame->result = chunks_record_RecordWriter_finish(&frame->writer); + frame->writer_live = false; + chunks_record_RecordWriter_drop(&frame->writer); +} + +/* txn_record_unlisted_pad frees what an unlist's encode holds live. */ +static void +txn_record_unlisted_pad(void *frame_) +{ + txn_record_unlisted_frame_t *frame = frame_; + + if (frame->bytes_live) + std_vec_free_Checked(&frame->bytes, 1, 1); + if (frame->writer_live) + chunks_record_RecordWriter_drop(&frame->writer); + frame->bytes_live = frame->writer_live = false; +} + +/* txn_record_CarryUnlisted_encode returns the record's body: the new storage (fields 1 to 3) and the ends of the carry's records (field 4). */ +void +txn_record_CarryUnlisted_encode(const TxnRecordCarryUnlisted *self, StdVec *result) +{ + txn_record_unlisted_frame_t frame = {.self = self, .result = result}; + + std_landing(&frame, txn_record_unlisted_body, txn_record_unlisted_pad); + txn_record_unlisted_pad(&frame); +} + +/* txn_record_CarryUnlisted_drop frees an unlist's ends. */ +void +txn_record_CarryUnlisted_drop(TxnRecordCarryUnlisted *self) +{ + std_vec_free_Checked(&self->ends, sizeof(uint64), _Alignof(uint64)); +} + +/* txn_record_CarryUnlisted_drop_void is CarryUnlisted's drop of a decode's result. */ +static void +txn_record_CarryUnlisted_drop_void(void *self) +{ + txn_record_CarryUnlisted_drop(self); +} + +/* txn_record_unlisted_read reads an unlist's body. */ +static bool +txn_record_unlisted_read(TxnRecordRead *read) +{ + static const uint32 allowed[] = {1, 2, 3, 4}; + TxnRecordCarryUnlisted *result = read->result; + ChunksRecordRecord *r = &read->record; + const uint8 *bytes; + size_t len; + size_t at = 0; + uint64 end = 0; + ChunksVarintMalformed malformed; + + if (!txn_record_known(read, allowed, lengthof(allowed)) || !txn_record_value(read) || + !txn_record_narrow(r, 1, &result->storage.tablespace, read->error) || + !txn_record_narrow(r, 2, &result->storage.database, read->error) || + !txn_record_narrow(r, 3, &result->storage.relation, read->error)) + return false; + bytes = chunks_record_Record_bytes(r, 4, &len); + if (bytes == NULL || len == 0) + return txn_record_refuse(read); + while (at < len) + { + uint64 distance; + + if (!chunks_varint_decode(bytes, len, &at, &distance, &malformed) || + (result->ends.len > 0 && distance == 0) || end + distance < end) + return txn_record_refuse(read); + end += distance; + if (!std_vec_try_reserve(&result->ends, 1, sizeof(uint64), _Alignof(uint64))) + { + *read->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(&result->ends, &end, sizeof(uint64), _Alignof(uint64)); + } + return true; +} + +/* + * txn_record_CarryUnlisted_decode reads an unlist's body, refusing any other + * form as data corruption: a field it doesn't know or lacks, a storage field + * past 32 bits, and ends that are empty, out of order, or cut short. + */ +bool +txn_record_CarryUnlisted_decode(const uint8 *body, size_t body_len, TxnRecordCarryUnlisted *result, + HostError *error) +{ + TxnRecordRead read = {.body = body, .body_len = body_len, .error = error, .result = result, + .drop = txn_record_CarryUnlisted_drop_void, .result_live = true, .read = txn_record_unlisted_read}; + + std_vec_new(&result->ends, sizeof(uint64), _Alignof(uint64)); + return txn_record_read(&read); +} + /* txn_record_put_runs appends runs of tuple ids as varints: each run's block, first offset, count, and bytes. */ static void txn_record_put_runs(const StdVec *runs, StdVec *out) @@ -2614,6 +2749,13 @@ txn_record_freed_body(void *frame_) chunks_record_RecordWriter_uint(w, 8, self->values.f0.relation); chunks_record_RecordWriter_field(w, 9, frame->bytes.ptr, frame->bytes.len); } + if (self->runs.len != 0 || self->dead.len != 0) + { + frame->bytes.len = 0; + for (size_t i = 0; i < self->enders.len; i++) + chunks_varint_put_varint(&frame->bytes, STD_VEC_AT(&self->enders, uint64, i)); + chunks_record_RecordWriter_field(w, 10, frame->bytes.ptr, frame->bytes.len); + } *frame->result = chunks_record_RecordWriter_finish(w); } @@ -2650,6 +2792,7 @@ txn_record_TidsFreed_drop(TxnRecordTidsFreed *self) { std_vec_free_Checked(&self->runs, TXN_RECORD_RUN); std_vec_free_Checked(&self->dead, TXN_RECORD_RUN); + std_vec_free_Checked(&self->enders, sizeof(uint64), _Alignof(uint64)); if (self->has_values) std_vec_free_Checked(&self->values.f1, TXN_RECORD_COUNTED); } @@ -2749,7 +2892,7 @@ txn_record_over(TxnRecordRead *read, const TxnRecordTidsFreed *result, uint64 pa static bool txn_record_freed_read(TxnRecordRead *read, uint16 most, uint64 page) { - static const uint32 allowed[] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + static const uint32 allowed[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; TxnRecordTidsFreed *result = read->result; ChunksRecordRecord *r = &read->record; size_t len; @@ -2808,6 +2951,30 @@ txn_record_freed_read(TxnRecordRead *read, uint16 most, uint64 page) } else return txn_record_refuse(read); + bytes = chunks_record_Record_bytes(r, 10, &len); + if ((bytes != NULL) != (result->runs.len != 0 || result->dead.len != 0)) + return txn_record_refuse(read); + if (bytes != NULL) + { + size_t at = 0; + uint64 tids = 0; + ChunksVarintMalformed malformed; + + for (size_t i = 0; i < result->runs.len; i++) + tids += STD_VEC_AT(&result->runs, std_TupleU32U16U16U64_t, i).f2; + for (size_t i = 0; i < result->dead.len; i++) + tids += STD_VEC_AT(&result->dead, std_TupleU32U16U16U64_t, i).f2; + while (at < len) + { + uint64 ender; + + if (result->enders.len == tids || !chunks_varint_decode(bytes, len, &at, &ender, &malformed)) + return txn_record_refuse(read); + std_vec_push_Checked(&result->enders, &ender, sizeof(uint64), _Alignof(uint64)); + } + if (result->enders.len != tids) + return txn_record_refuse(read); + } return txn_record_uint(r, 4, &result->horizon, read->error); } @@ -2832,8 +2999,10 @@ txn_record_freed_read_at(TxnRecordRead *read) * txn_record_TidsFreed_decode reads a free's body, refusing any other form as * data corruption: a field it doesn't know or lacks, a field 6 with no run, a * run runs refuses, a block whose runs of both fields give back more bytes - * than a page holds, fields 7 to 9 not all present or all absent, and values - * that are empty, out of id order, or of a count of zero. + * than a page holds, fields 7 to 9 not all present or all absent, values + * that are empty, out of id order, or of a count of zero, and a field 10 + * present without a run in field 5 or 6, absent with one, or of another + * count of enders than their tuple ids. */ bool txn_record_TidsFreed_decode(const uint8 *body, size_t body_len, uint16 most, uint64 page, TxnRecordTidsFreed *result, @@ -2845,6 +3014,7 @@ txn_record_TidsFreed_decode(const uint8 *body, size_t body_len, uint16 most, uin std_vec_new(&result->runs, TXN_RECORD_RUN); std_vec_new(&result->dead, TXN_RECORD_RUN); + std_vec_new(&result->enders, sizeof(uint64), _Alignof(uint64)); result->has_values = false; return txn_record_read(&freed.read); } @@ -3234,6 +3404,13 @@ txn_record_values_Debug_fmt(const void *value, StdFormatter *f) return std_tuple_Debug_fmt(&tuple, f); } +/* txn_record_u64_eq reports whether two u64s are equal, an element's PartialEq. */ +static bool +txn_record_u64_eq(const void *a, const void *b) +{ + return *(const uint64 *) a == *(const uint64 *) b; +} + /* txn_record_TidsFreed_PartialEq_eq reports whether two frees are equal field by field. */ bool txn_record_TidsFreed_PartialEq_eq(const TxnRecordTidsFreed *self, const TxnRecordTidsFreed *other) @@ -3241,6 +3418,7 @@ txn_record_TidsFreed_PartialEq_eq(const TxnRecordTidsFreed *self, const TxnRecor return host_Storage_PartialEq_eq(&self->storage, &other->storage) && self->horizon == other->horizon && std_vec_PartialEq_eq(&self->runs, &other->runs, sizeof(std_TupleU32U16U16U64_t), txn_record_run_eq) && std_vec_PartialEq_eq(&self->dead, &other->dead, sizeof(std_TupleU32U16U16U64_t), txn_record_run_eq) && + std_vec_PartialEq_eq(&self->enders, &other->enders, sizeof(uint64), txn_record_u64_eq) && self->has_values == other->has_values && (!self->has_values || (host_Storage_PartialEq_eq(&self->values.f0, &other->values.f0) && @@ -3254,6 +3432,7 @@ txn_record_TidsFreed_Debug_fmt(const void *value, StdFormatter *f) const TxnRecordTidsFreed *self = value; TxnRecordList runs = {&self->runs, sizeof(std_TupleU32U16U16U64_t), txn_record_run_Debug_fmt}; TxnRecordList dead = {&self->dead, sizeof(std_TupleU32U16U16U64_t), txn_record_run_Debug_fmt}; + TxnRecordList enders = {&self->enders, sizeof(uint64), std_u64_Debug_fmt}; StdOptionRef values = {self->has_values, {&self->values, txn_record_values_Debug_fmt}}; StdDebugStruct s = std_fmt_debug_struct(f, "TidsFreed", 9); @@ -3261,6 +3440,1662 @@ txn_record_TidsFreed_Debug_fmt(const void *value, StdFormatter *f) std_DebugStruct_field(&s, "horizon", 7, &self->horizon, std_u64_Debug_fmt); std_DebugStruct_field(&s, "runs", 4, &runs, txn_record_List_Debug_fmt); std_DebugStruct_field(&s, "dead", 4, &dead, txn_record_List_Debug_fmt); + std_DebugStruct_field(&s, "enders", 6, &enders, txn_record_List_Debug_fmt); std_DebugStruct_field(&s, "values", 6, &values, std_option_Debug_fmt); return std_DebugStruct_finish(&s); } + +#ifdef DOLTGRES_TEST_TXN +/* TXN_RECORD_TESTS_UNWRAP is Result::unwrap's panic text. */ +#define TXN_RECORD_TESTS_UNWRAP "called `Result::unwrap()` on an `Err` value" + +/* TxnRecordTestsRead is what a decode of a body came to: read, refused as data corruption, or failed otherwise. */ +typedef enum TxnRecordTestsRead +{ + TXN_RECORD_TESTS_READ, + TXN_RECORD_TESTS_CORRUPT, + TXN_RECORD_TESTS_FAILED, +} TxnRecordTestsRead; + +/* TxnRecordTestsField is a field a test writes: its number and its integer, or its bytes when they are set. */ +typedef struct TxnRecordTestsField +{ + uint32 number; + uint64 value; + const void *bytes; + size_t bytes_len; +} TxnRecordTestsField; + +/* TxnRecordTestsU32s is a slice of u32s, as a test's message shows it. */ +typedef struct TxnRecordTestsU32s +{ + const uint32 *ptr; + size_t len; +} TxnRecordTestsU32s; + +/* txn_record_tests_u32s_Debug_fmt is a slice of u32s' Debug. */ +static bool +txn_record_tests_u32s_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnRecordTestsU32s *u32s = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t i = 0; i < u32s->len; i++) + std_DebugList_entry(&list, &u32s->ptr[i], std_u32_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_record_tests_vec_Debug_fmt is a Vec's Debug. */ +static bool +txn_record_tests_vec_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *vec = value; + StdBytes bytes = {vec->ptr, vec->len}; + + return std_bytes_Debug_fmt(&bytes, f); +} + +/* txn_record_tests_piece_Debug_fmt is a piece's Debug, its key and its value. */ +static bool +txn_record_tests_piece_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_TupleVecU8VecU8_t *piece = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&piece->f0, txn_record_tests_vec_Debug_fmt), + STD_ITEM(&piece->f1, txn_record_tests_vec_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_record_tests_pieces_Debug_fmt is a Vec of pieces' Debug. */ +static bool +txn_record_tests_pieces_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *pieces = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t i = 0; i < pieces->len; i++) + std_DebugList_entry(&list, &STD_VEC_AT(pieces, txn_TupleVecU8VecU8_t, i), txn_record_tests_piece_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_record_tests_bytes_eq returns whether a Vec holds the bytes given. */ +static bool +txn_record_tests_bytes_eq(const StdVec *vec, const void *bytes, size_t bytes_len) +{ + return vec->len == bytes_len && (bytes_len == 0 || memcmp(vec->ptr, bytes, bytes_len) == 0); +} + +/* txn_record_tests_message_decoded panics at a line unless refused, its message a prefix, an argument, and " decoded". */ +static void +txn_record_tests_message_decoded(bool refused, const char *prefix, const void *shown, StdFmtFn shown_fmt, uint32 line, + uint32 col) +{ + if (!refused) + { + StdFmtPiece pieces[] = {{prefix, strlen(prefix), 0, {0}}, STD_FMT_ARG(0), STD_FMT_LIT(" decoded")}; + StdFmtArg args[] = {{shown, shown_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + std_panic_fmt(RUST_FILE, line, col, &arguments); + } +} + +/* txn_record_tests_fields_decoded panics at a line unless a body of the numbered fields was refused, naming them. */ +static void +txn_record_tests_fields_decoded(bool refused, const uint32 *numbers, size_t numbers_len, uint32 line) +{ + TxnRecordTestsU32s shown = {numbers, numbers_len}; + + txn_record_tests_message_decoded(refused, "fields ", &shown, txn_record_tests_u32s_Debug_fmt, line, 13); +} + +/* txn_record_tests_write appends fields to a writer, each an integer or, with bytes, a field of them. */ +static void +txn_record_tests_write(ChunksRecordRecordWriter *w, const TxnRecordTestsField *fields, size_t fields_len) +{ + for (size_t i = 0; i < fields_len; i++) + { + if (fields[i].bytes != NULL) + chunks_record_RecordWriter_field(w, fields[i].number, fields[i].bytes, fields[i].bytes_len); + else + chunks_record_RecordWriter_uint(w, fields[i].number, fields[i].value); + } +} + +/* txn_record_tests_body returns a record of fields. */ +static StdVec +txn_record_tests_body(const TxnRecordTestsField *fields, size_t fields_len) +{ + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + txn_record_tests_write(&w, fields, fields_len); + return chunks_record_RecordWriter_finish(&w); +} + +/* txn_record_tests_status returns what a decode came to, freeing its error and then the body. */ +static TxnRecordTestsRead +txn_record_tests_status(bool ok, HostError *error, StdVec *body) +{ + TxnRecordTestsRead read = TXN_RECORD_TESTS_READ; + + if (!ok) + { + read = memcmp(host_Error_sqlstate(error), "XX001", 5) == 0 ? TXN_RECORD_TESTS_CORRUPT : TXN_RECORD_TESTS_FAILED; + host_Error_drop(error); + } + std_vec_free_Checked(body, 1, 1); + return read; +} + +/* txn_record_tests_change_read decodes a row change's body less its last cut bytes, freeing the body. */ +static TxnRecordTestsRead +txn_record_tests_change_read(StdVec body, size_t cut) +{ + TxnRecordRowChange read; + HostError error; + bool ok = txn_record_RowChange_decode(body.ptr, body.len - cut, &read, &error); + + if (ok) + txn_record_RowChange_drop(&read); + return txn_record_tests_status(ok, &error, &body); +} + +/* txn_record_tests_left_read decodes a leave's body less its last cut bytes, freeing the body. */ +static TxnRecordTestsRead +txn_record_tests_left_read(StdVec body, size_t cut) +{ + TxnRecordPageLeft read; + HostError error; + bool ok = txn_record_PageLeft_decode(body.ptr, body.len - cut, &read, &error); + + return txn_record_tests_status(ok, &error, &body); +} + +/* txn_record_tests_chunk_read decodes a chunk's body less its last cut bytes, freeing the body. */ +static TxnRecordTestsRead +txn_record_tests_chunk_read(StdVec body, size_t cut) +{ + TxnRecordChunkWrite read; + HostError error; + bool ok = txn_record_ChunkWrite_decode(body.ptr, body.len - cut, &read, &error); + + if (ok) + txn_record_ChunkWrite_drop(&read); + return txn_record_tests_status(ok, &error, &body); +} + +/* txn_record_tests_carried_read decodes a carry's body, freeing the body. */ +static TxnRecordTestsRead +txn_record_tests_carried_read(StdVec body) +{ + TxnRecordMapCarried read; + HostError error; + bool ok = txn_record_MapCarried_decode(body.ptr, body.len, &read, &error); + + if (ok) + txn_record_MapCarried_drop(&read); + return txn_record_tests_status(ok, &error, &body); +} + +/* txn_record_tests_advanced_read decodes a fold's body less its last cut bytes, freeing the body. */ +static TxnRecordTestsRead +txn_record_tests_advanced_read(StdVec body, size_t cut) +{ + TxnRecordBaseAdvanced read; + HostError error; + bool ok = txn_record_BaseAdvanced_decode(body.ptr, body.len - cut, &read, &error); + + if (ok) + txn_record_BaseAdvanced_drop(&read); + return txn_record_tests_status(ok, &error, &body); +} + +/* txn_record_tests_freed_read decodes a free's body in a page of 291 tuples and 8168 bytes, freeing the body. */ +static TxnRecordTestsRead +txn_record_tests_freed_read(StdVec body) +{ + TxnRecordTidsFreed read; + HostError error; + bool ok = txn_record_TidsFreed_decode(body.ptr, body.len, 291, 8168, &read, &error); + + if (ok) + txn_record_TidsFreed_drop(&read); + return txn_record_tests_status(ok, &error, &body); +} + +/* txn_record_tests_change_round asserts at a line that a row change reads back as it was written. */ +static void +txn_record_tests_change_round(const TxnRecordRowChange *written, uint32 line) +{ + StdVec body; + TxnRecordRowChange read; + HostError error; + + txn_record_RowChange_encode(written, &body); + if (!txn_record_RowChange_decode(body.ptr, body.len, &read, &error)) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_RECORD_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT_EQ(line, 0, txn_record_RowChange_PartialEq_eq(&read, written), &read, txn_record_RowChange_Debug_fmt, + written, txn_record_RowChange_Debug_fmt); + txn_record_RowChange_drop(&read); + std_vec_free_Checked(&body, 1, 1); +} + +/* txn_record_tests_left_round asserts at a line that a leave reads back as it was written. */ +static void +txn_record_tests_left_round(const TxnRecordPageLeft *written, uint32 line) +{ + StdVec body; + TxnRecordPageLeft read; + HostError error; + + txn_record_PageLeft_encode(written, &body); + if (!txn_record_PageLeft_decode(body.ptr, body.len, &read, &error)) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_RECORD_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT_EQ(line, 0, txn_record_PageLeft_PartialEq_eq(&read, written), &read, txn_record_PageLeft_Debug_fmt, + written, txn_record_PageLeft_Debug_fmt); + std_vec_free_Checked(&body, 1, 1); +} + +/* txn_record_tests_chunk_decoded returns a chunk's encoding read back, asserting at a line that it is the chunk. */ +static TxnRecordChunkWrite +txn_record_tests_chunk_decoded(const TxnRecordChunkWrite *written, uint32 line) +{ + StdVec body; + TxnRecordChunkWrite read; + HostError error; + + txn_record_ChunkWrite_encode(written, &body); + if (!txn_record_ChunkWrite_decode(body.ptr, body.len, &read, &error)) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_RECORD_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT_EQ(line, 0, txn_record_ChunkWrite_PartialEq_eq(&read, written), &read, txn_record_ChunkWrite_Debug_fmt, + written, txn_record_ChunkWrite_Debug_fmt); + std_vec_free_Checked(&body, 1, 1); + return read; +} + +/* txn_record_tests_carried_round asserts at a line that a carry reads back as it was written. */ +static void +txn_record_tests_carried_round(const TxnRecordMapCarried *written, uint32 line) +{ + StdVec body; + TxnRecordMapCarried read; + HostError error; + + txn_record_MapCarried_encode(written, &body); + if (!txn_record_MapCarried_decode(body.ptr, body.len, &read, &error)) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_RECORD_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT_EQ(line, 0, txn_record_MapCarried_PartialEq_eq(&read, written), &read, txn_record_MapCarried_Debug_fmt, + written, txn_record_MapCarried_Debug_fmt); + txn_record_MapCarried_drop(&read); + std_vec_free_Checked(&body, 1, 1); +} + +/* txn_record_tests_advanced_round asserts at a line that a fold's record reads back as it was written. */ +static void +txn_record_tests_advanced_round(const TxnRecordBaseAdvanced *written, uint32 line) +{ + StdVec body; + TxnRecordBaseAdvanced read; + HostError error; + + txn_record_BaseAdvanced_encode(written, &body); + if (!txn_record_BaseAdvanced_decode(body.ptr, body.len, &read, &error)) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_RECORD_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT_EQ(line, 0, txn_record_BaseAdvanced_PartialEq_eq(&read, written), &read, + txn_record_BaseAdvanced_Debug_fmt, written, txn_record_BaseAdvanced_Debug_fmt); + txn_record_BaseAdvanced_drop(&read); + std_vec_free_Checked(&body, 1, 1); +} + +/* txn_record_tests_freed_round asserts at a line that a free reads back as it was written in a page of 291 tuples. */ +static void +txn_record_tests_freed_round(const TxnRecordTidsFreed *written, uint32 line) +{ + StdVec body; + TxnRecordTidsFreed read; + HostError error; + + txn_record_TidsFreed_encode(written, &body); + if (!txn_record_TidsFreed_decode(body.ptr, body.len, 291, 8168, &read, &error)) + STD_RESULT_EXPECT_FAILED(line, 0, TXN_RECORD_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT_EQ(line, 0, txn_record_TidsFreed_PartialEq_eq(&read, written), &read, txn_record_TidsFreed_Debug_fmt, + written, txn_record_TidsFreed_Debug_fmt); + txn_record_TidsFreed_drop(&read); + std_vec_free_Checked(&body, 1, 1); +} + +/* txn_record_tests_ids sets a put's value ids. */ +static void +txn_record_tests_ids(TxnMemtablePut *put, const uint32 *ids, size_t ids_len) +{ + std_vec_free_Checked(&put->values, sizeof(uint32), _Alignof(uint32)); + std_slice_to_vec_Checked(&put->values, ids, ids_len, sizeof(uint32), _Alignof(uint32)); +} + +/* txn_record_tests_put returns a put of a value with an address and a tuple id. */ +static TxnMemtablePut +txn_record_tests_put(uint64 tuple) +{ + TxnMemtablePut put = {.tuple = {tuple}, .heap_bytes = 36}; + chunks_TupleU32Locator_t address = {2, chunks_trailer_Locator_at(7, 640, 64)}; + + std_slice_to_vec_Checked(&put.value, "row", 3, 1, 1); + std_slice_to_vec_Checked(&put.addresses, &address, 1, sizeof(address), _Alignof(chunks_TupleU32Locator_t)); + std_vec_new(&put.values, sizeof(uint32), _Alignof(uint32)); + return put; +} + +/* txn_record_tests_op returns the which'th op a row change's round trip writes, in the order its test lists them. */ +static TxnMemtableOp +txn_record_tests_op(int which) +{ + TxnMemtableOp op = {0}; + + switch (which) + { + case 0: + op.tag = TXN_MEMTABLE_OP_PUT; + op.u.put = txn_record_tests_put(1 << 16 | 1); + break; + case 1: + op.tag = TXN_MEMTABLE_OP_DELETE; + break; + case 2: + op.tag = TXN_MEMTABLE_OP_MOVE; + op.u.move.tag = TXN_MEMTABLE_SUCCESSOR_KEY; + std_slice_to_vec_Checked(&op.u.move.u.key, "m", 1, 1, 1); + break; + case 3: + op.tag = TXN_MEMTABLE_OP_ADD; + op.u.add = txn_record_tests_put(2); + break; + case 4: + op.tag = TXN_MEMTABLE_OP_REMOVE; + op.u.remove.f0.f0 = 7; + op.u.remove.has_f1 = true; + op.u.remove.f1.tag = TXN_MEMTABLE_SUCCESSOR_OTHER_TABLE; + break; + case 5: + op.tag = TXN_MEMTABLE_OP_ADD_ROW; + op.u.add_row = txn_record_tests_put(3); + break; + case 6: + op.tag = TXN_MEMTABLE_OP_UPDATE_ROW; + op.u.update_row.f0.f0 = 3; + op.u.update_row.f1 = txn_record_tests_put(4); + break; + default: + op.tag = TXN_MEMTABLE_OP_DELETE_ROW; + op.u.delete_row.f0.f0 = 4; + break; + } + return op; +} + +/* txn_record_tests_change returns a row change of an op by a writer, in a storage of a table, top zero for none. */ +static TxnRecordRowChange +txn_record_tests_change(TxnMemtableOp op, uint64 top) +{ + TxnRecordRowChange change = { + .writer = {41, top != 0, top, 3}, + .storage = {1663, 5, 16384}, + .table = 16385, + .op = op, + }; + + if (op.tag == TXN_MEMTABLE_OP_PUT || op.tag == TXN_MEMTABLE_OP_ADD || op.tag == TXN_MEMTABLE_OP_ADD_ROW || + op.tag == TXN_MEMTABLE_OP_UPDATE_ROW) + change.heap_bytes = 36; + std_slice_to_vec_Checked(&change.key, "k", 1, 1, 1); + std_vec_new(&change.blobs, sizeof(chunks_TupleAddressU64_t), _Alignof(chunks_TupleAddressU64_t)); + std_vec_new(&change.kept, sizeof(size_t), _Alignof(size_t)); + return change; +} + +/* txn_record_tests_put_change returns a row change of a put. */ +static TxnRecordRowChange +txn_record_tests_put_change(TxnMemtablePut put, uint64 top) +{ + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_PUT, .u.put = put}; + + return txn_record_tests_change(op, top); +} + +/* txn_record_tests_blob appends a stored value's root, the address of its bytes, and its length to a change. */ +static void +txn_record_tests_blob(TxnRecordRowChange *change, const char *bytes, uint64 length) +{ + chunks_TupleAddressU64_t blob = {chunks_address_Address_of((const uint8 *) bytes, strlen(bytes)), length}; + + std_vec_push_Checked(&change->blobs, &blob, sizeof(blob), _Alignof(chunks_TupleAddressU64_t)); +} + +/* txn_record_tests_kept sets a change's kept places. */ +static void +txn_record_tests_kept(TxnRecordRowChange *change, const size_t *kept, size_t kept_len) +{ + std_vec_free_Checked(&change->kept, sizeof(size_t), _Alignof(size_t)); + std_slice_to_vec_Checked(&change->kept, kept, kept_len, sizeof(size_t), _Alignof(size_t)); +} + +/* txn_record_tests_valued returns a put's body with heap bytes and each numbered field, 24 a stored value and others 9. */ +static StdVec +txn_record_tests_valued(const uint32 *numbers, size_t numbers_len) +{ + static const uint8 stored[34] = {7,[33] = 1}; + const TxnRecordTestsField fields[] = {{1, 41}, {2, 1}, {3, 5}, {4, 1}, {6, 0, "k", 1}, {7, 0, "row", 3}, {9, 1}, + {15, 1}, {16, 0}, {22, 36}}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + txn_record_tests_write(&w, fields, lengthof(fields)); + for (size_t i = 0; i < numbers_len; i++) + { + if (numbers[i] == 24) + chunks_record_RecordWriter_field(&w, 24, stored, sizeof(stored)); + else + chunks_record_RecordWriter_uint(&w, numbers[i], 9); + } + return chunks_record_RecordWriter_finish(&w); +} + +/* + * test_txn_record_tests_a_row_change_reads_back_and_refuses_every_other_form: + * Doltgrust's own, ported: each op a row change carries reads back as it was + * written, with its writer, the storage and table, the key, a + * subtransaction's xid beside its top-level one, the no-key and frozen marks, + * and a put's heap bytes and value ids (fields 22 and 24, the fork's own), + * and any other form is data corruption, a put without its heap bytes, a + * delete with them or with value ids, and an empty list of ids among them. + */ +void +test_txn_record_tests_a_row_change_reads_back_and_refuses_every_other_form(void) +{ + static const uint32 marked_ids[] = {16390, 1U << 31}; + static const uint8 one_id[] = {7}; + static const uint8 cut_value[] = {7, 0, 0}; + static const uint32 all[] = {24, 25, 26, 27}; + static const uint32 first[] = {24}; + static const uint32 second[] = {24, 25, 26}; + static const uint32 third[] = {25, 26, 27}; + static const uint32 fourth[] = {25}; + static const uint32 fifth[] = {27}; + static const TxnRecordTestsU32s refused_sets[] = {{first, 1}, {second, 3}, {third, 3}, {fourth, 1}, {fifth, 1}}; + const TxnRecordTestsField added[] = {{1, 41}, {2, 1}, {3, 5}, {4, 1}, {6, 0, "k", 1}, {11, TXN_RECORD_ADD}, + {15, 1}, {16, 0}}; + const TxnRecordTestsField wide[] = {{1, 41}, {2, UINT64CONST(1) << 40}, {3, 5}, {4, 1}, {6, 0, "k", 1}, {15, 1}, + {16, 0}}; + const TxnRecordTestsField unweighed[] = {{1, 41}, {2, 1}, {3, 5}, {4, 1}, {6, 0, "k", 1}, {7, 0, "row", 3}, {9, 1}, + {15, 1}, {16, 0}}; + const TxnRecordTestsField weighed[] = {{1, 41}, {2, 1}, {3, 5}, {4, 1}, {6, 0, "k", 1}, {15, 1}, {16, 0}, + {22, 36}}; + const TxnRecordTestsField delete_valued[] = {{1, 41}, {2, 1}, {3, 5}, {4, 1}, {6, 0, "k", 1}, {15, 1}, {16, 0}, + {24, 0, one_id, 1}}; + const TxnRecordTestsField emptied[] = {{1, 41}, {2, 1}, {3, 5}, {4, 1}, {6, 0, "k", 1}, {7, 0, "row", 3}, {9, 1}, + {15, 1}, {16, 0}, {22, 36}, {24, 0, "", 0}}; + const TxnRecordTestsField cut[] = {{1, 41}, {2, 1}, {3, 5}, {4, 1}, {6, 0, "k", 1}, {7, 0, "row", 3}, {9, 1}, + {15, 1}, {16, 0}, {22, 36}, {24, 0, cut_value, sizeof(cut_value)}, {25, 1}, {26, 2}, {27, 3}}; + TxnMemtablePut put = txn_record_tests_put(5); + TxnRecordRowChange marked; + TxnRecordRowChange ended; + TxnRecordRowChange whole; + StdVec body; + + for (int which = 0; which < 8; which++) + { + for (uint64 top = 0; top <= 40; top += 40) + { + TxnRecordRowChange written = txn_record_tests_change(txn_record_tests_op(which), top); + + txn_record_tests_change_round(&written, __LINE__); + txn_record_RowChange_drop(&written); + } + } + txn_record_tests_ids(&put, marked_ids, lengthof(marked_ids)); + marked = txn_record_tests_put_change(put, 0); + marked.no_key = true; + marked.frozen = true; + marked.has_toast = true; + marked.toast = (TxnRecordToast) + { + 1663, 16391, 16392 + }; + txn_record_tests_blob(&marked, "one", 4000); + txn_record_tests_blob(&marked, "two", 300000); + marked.has_replaced = true; + marked.replaced.f0 = 3 << 16 | 7; + txn_record_tests_change_round(&marked, __LINE__); + txn_record_RowChange_drop(&marked); + ended = txn_record_tests_change(txn_record_tests_op(1), 0); + ended.has_replaced = true; + ended.replaced.f0 = 7; + txn_record_RowChange_encode(&ended, &body); + txn_record_RowChange_drop(&ended); + STD_ASSERT(__LINE__, 9, txn_record_tests_change_read(body, 0) == TXN_RECORD_TESTS_CORRUPT, + "RowChange::decode(&ended.encode()).unwrap_err().sqlstate() == b\"XX001\""); + whole = txn_record_tests_put_change(txn_record_tests_put(1), 0); + txn_record_RowChange_encode(&whole, &body); + txn_record_RowChange_drop(&whole); + STD_ASSERT(__LINE__, 9, txn_record_tests_change_read(body, 1) == TXN_RECORD_TESTS_CORRUPT, + "refused(&body[..body.len() - 1])"); + STD_ASSERT(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_body(added, lengthof(added)), 0) == + TXN_RECORD_TESTS_CORRUPT, "refused(&w.finish())"); + STD_ASSERT(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_body(wide, lengthof(wide)), 0) == + TXN_RECORD_TESTS_CORRUPT, "refused(&w.finish())"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_body(unweighed, lengthof(unweighed)), 0) == + TXN_RECORD_TESTS_CORRUPT, "a put without its heap bytes decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_body(weighed, lengthof(weighed)), 0) == + TXN_RECORD_TESTS_CORRUPT, "a delete with heap bytes decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_body(delete_valued, lengthof(delete_valued)), 0) == + TXN_RECORD_TESTS_CORRUPT, "a delete with value ids decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_body(emptied, lengthof(emptied)), 0) == + TXN_RECORD_TESTS_CORRUPT, "an empty field 24 decoded"); + STD_ASSERT(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_valued(all, lengthof(all)), 0) == TXN_RECORD_TESTS_READ, + "RowChange::decode(&valued(&[24, 25, 26, 27])).is_ok()"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_body(cut, lengthof(cut)), 0) == + TXN_RECORD_TESTS_CORRUPT, "a value with a cut address decoded"); + for (size_t i = 0; i < lengthof(refused_sets); i++) + txn_record_tests_fields_decoded(txn_record_tests_change_read(txn_record_tests_valued(refused_sets[i].ptr, + refused_sets[i].len), + 0) == TXN_RECORD_TESTS_CORRUPT, + refused_sets[i].ptr, refused_sets[i].len, __LINE__); +} + +/* txn_record_tests_left returns a leave's body to a counter, with an extra field of 1 unless its number is zero. */ +static StdVec +txn_record_tests_left(uint64 to, uint32 extra) +{ + const TxnRecordTestsField fields[] = {{1, 41}, {2, 1663}, {3, 5}, {4, 16384}, {6, to}}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + txn_record_tests_write(&w, fields, lengthof(fields)); + if (extra != 0) + chunks_record_RecordWriter_uint(&w, extra, 1); + chunks_record_RecordWriter_uint(&w, 16, 3); + return chunks_record_RecordWriter_finish(&w); +} + +/* + * test_txn_record_tests_a_page_leave_reads_back_and_refuses_every_other_form: + * the fork's own (CommPF 5's ruling on a fresh page's leave): a leave reads + * back as it was written, with its writer and a subtransaction's xid, its + * storage, and the counter it moved to, and a field it doesn't know, a + * counter that names no page's first tuple id, and a cut body are data + * corruption. + */ +void +test_txn_record_tests_a_page_leave_reads_back_and_refuses_every_other_form(void) +{ + for (uint64 top = 0; top <= 40; top += 40) + { + TxnRecordPageLeft left = {.writer = {41, top != 0, top, 3}, .storage = {1663, 5, 16384}, .to = 3 << 16 | 1}; + + txn_record_tests_left_round(&left, __LINE__); + } + STD_ASSERT(__LINE__, 9, txn_record_tests_left_read(txn_record_tests_left(1 << 16 | 1, 0), 0) == + TXN_RECORD_TESTS_READ, "PageLeft::decode(&left(1 << 16 | 1, None)).is_ok()"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_left_read(txn_record_tests_left(1 << 16 | 2, 0), 0) == + TXN_RECORD_TESTS_CORRUPT, "a counter past a page's first offset decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_left_read(txn_record_tests_left(1, 0), 0) == + TXN_RECORD_TESTS_CORRUPT, "a leave to block 0 decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_left_read(txn_record_tests_left(1 << 16 | 1, 9), 0) == + TXN_RECORD_TESTS_CORRUPT, "an unknown field decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_left_read(txn_record_tests_left(1 << 16 | 1, 0), 1) == + TXN_RECORD_TESTS_CORRUPT, "a cut leave decoded"); +} + +/* txn_record_tests_leaf returns a leaf of the entries a=1 and, of two, b=2, with no addresses. */ +static StdVec +txn_record_tests_leaf(size_t entries) +{ + const StdBytes keys[] = {{(const uint8 *) "a", 1}, {(const uint8 *) "b", 1}}; + const StdBytes values[] = {{(const uint8 *) "1", 1}, {(const uint8 *) "2", 1}}; + const StdVec addresses[] = {STD_VEC_NEW, STD_VEC_NEW}; + StdVec leaf; + + trees_node_encode_leaf(keys, entries, values, entries, addresses, entries, false, &leaf); + return leaf; +} + +/* txn_record_tests_stamps returns the stamps of transaction 7's tuples 1 to count, one an entry. */ +static ChunksTrailerStamps +txn_record_tests_stamps(uint64 count) +{ + StdVec single; + + std_vec_new(&single, sizeof(ChunksTrailerStamp), _Alignof(ChunksTrailerStamp)); + for (uint64 tuple = 1; tuple <= count; tuple++) + { + ChunksTrailerStamp stamp = {7, tuple}; + + std_vec_push_Checked(&single, &stamp, sizeof(stamp), _Alignof(ChunksTrailerStamp)); + } + return chunks_trailer_Stamps_FromVecStamp_from(&single); +} + +/* txn_record_tests_node returns a built base's node of a chunk's bytes with stamps stored alone, in a storage at a point. */ +static TxnRecordChunkWrite +txn_record_tests_node(const StdVec *bytes, const ChunksTrailerStamps *stamps, uint32 relation, uint64 point) +{ + TxnRecordChunkWrite chunk = { + .repository = 16384, + .address = chunks_address_Address_of(bytes->ptr, bytes->len), + .has_node = true, + .node = {.tablespace = 1663, .relation = relation, .point = point, .writer = UINT64CONST(0x1000002f1)}, + }; + + std_vec_clone(&chunk.bytes, bytes, 1, 1); + chunks_trailer_store((const uint8 *) "", 0, NULL, 0, stamps, &chunk.node.stamps); + return chunk; +} + +/* txn_record_tests_pieces_drop frees a chunk's pieces. */ +static void +txn_record_tests_pieces_drop(StdVec *pieces) +{ + for (size_t i = 0; i < pieces->len; i++) + { + std_vec_free_Checked(&STD_VEC_AT(pieces, txn_TupleVecU8VecU8_t, i).f0, 1, 1); + std_vec_free_Checked(&STD_VEC_AT(pieces, txn_TupleVecU8VecU8_t, i).f1, 1, 1); + } + std_vec_free_Checked(pieces, sizeof(txn_TupleVecU8VecU8_t), _Alignof(txn_TupleVecU8VecU8_t)); +} + +/* txn_record_tests_be appends an integer's low bytes, most significant first. */ +static void +txn_record_tests_be(StdVec *out, uint64 value, size_t bytes) +{ + for (size_t i = bytes; i > 0; i--) + { + uint8 byte = (uint8) (value >> (8 * (i - 1))); + + std_vec_push_Checked(out, &byte, 1, 1); + } +} + +/* + * test_txn_record_tests_a_built_bases_node_reads_back_with_its_stamps: PLAN's + * Chunk record row, field 5 (the rewrite text's standby ruling): a built + * base's leaf reads back with its stamps, and its stored form in the working + * state's delta is one piece under its key: its writer, its length, and its + * bytes and their trailer with no locators. + */ +void +test_txn_record_tests_a_built_bases_node_reads_back_with_its_stamps(void) +{ + StdVec leaf = txn_record_tests_leaf(2); + ChunksTrailerStamps stamps = txn_record_tests_stamps(2); + TxnRecordChunkWrite chunk = txn_record_tests_node(&leaf, &stamps, 16390, UINT64CONST(0x100000028)); + TxnRecordChunkWrite read = txn_record_tests_chunk_decoded(&chunk, __LINE__); + txn_TupleVecU8VecU8_t piece; + StdVec value; + StdVec pieces; + StdVec wanted = {&piece, 1, 1}; + HostError error; + uint8 node_key[TXN_MEMTABLE_BRANCH_NODE_KEY]; + + chunks_trailer_store(leaf.ptr, leaf.len, NULL, 0, &stamps, &value); + txn_memtable_branch_node_key(chunk.address.f0, 1663, 16390, UINT64CONST(0x100000028), node_key); + std_slice_to_vec_Checked(&piece.f0, node_key, sizeof(node_key), 1, 1); + txn_record_tests_be(&piece.f0, 0, 2); + std_vec_new(&piece.f1, 1, 1); + txn_record_tests_be(&piece.f1, UINT64CONST(0x1000002f1), 8); + txn_record_tests_be(&piece.f1, value.len, 4); + std_vec_extend_from_slice_Checked(&piece.f1, value.ptr, value.len, 1, 1); + if (!txn_record_ChunkWrite_pieces(&read, &pieces, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, TXN_RECORD_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + STD_ASSERT_EQ(__LINE__, 9, pieces.len == 1 && + txn_record_tests_bytes_eq(&STD_VEC_AT(&pieces, txn_TupleVecU8VecU8_t, 0).f0, piece.f0.ptr, + piece.f0.len) && + txn_record_tests_bytes_eq(&STD_VEC_AT(&pieces, txn_TupleVecU8VecU8_t, 0).f1, piece.f1.ptr, + piece.f1.len), + &pieces, txn_record_tests_pieces_Debug_fmt, &wanted, txn_record_tests_pieces_Debug_fmt); + txn_record_tests_pieces_drop(&pieces); + std_vec_free_Checked(&piece.f0, 1, 1); + std_vec_free_Checked(&piece.f1, 1, 1); + std_vec_free_Checked(&value, 1, 1); + txn_record_ChunkWrite_drop(&chunk); + chunks_trailer_Stamps_drop(&stamps); + std_vec_free_Checked(&leaf, 1, 1); +} + +/* txn_record_tests_stamped_refused returns whether a built node of bytes with stamps is refused as data corruption. */ +static bool +txn_record_tests_stamped_refused(const StdVec *bytes, const ChunksTrailerStamps *stamps) +{ + TxnRecordChunkWrite chunk = txn_record_tests_node(bytes, stamps, 16390, UINT64CONST(0x100000028)); + StdVec body; + + txn_record_ChunkWrite_encode(&chunk, &body); + txn_record_ChunkWrite_drop(&chunk); + return txn_record_tests_chunk_read(body, 0) == TXN_RECORD_TESTS_CORRUPT; +} + +/* + * test_txn_record_tests_stamps_on_a_blob_or_of_another_count_are_refused: + * PLAN's Chunk record row, field 5: stamps on a chunk that isn't a tree's + * node, a blob's leaf, and stamps whose count isn't the node's entries' are + * data corruption. + */ +void +test_txn_record_tests_stamps_on_a_blob_or_of_another_count_are_refused(void) +{ + static const uint8 raw[] = {CHUNKS_HEADER_BLOB_LEAF, 0, 0, 'r', 'a', 'w'}; + ChunksTrailerStamps one = txn_record_tests_stamps(1); + StdVec blob; + StdVec leaf = txn_record_tests_leaf(2); + + std_slice_to_vec_Checked(&blob, raw, sizeof(raw), 1, 1); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_stamped_refused(&blob, &one), "stamps on a blob's leaf decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_stamped_refused(&leaf, &one), + "one stamp on a leaf of two entries decoded"); + std_vec_free_Checked(&leaf, 1, 1); + std_vec_free_Checked(&blob, 1, 1); + chunks_trailer_Stamps_drop(&one); +} + +/* + * txn_record_tests_take returns a carried version: its writer, its tuple id, + * its entry unless value is NULL, its ender, and its values mark unless mark + * is NULL. + */ +static TxnRecordCarriedTake +txn_record_tests_take(TxnMemtableVersionWriter writer, uint64 tuple, const uint8 *value, size_t value_len, + uint64 ender, const uint32 *mark, size_t mark_len) +{ + TxnRecordCarriedTake take = {.writer = writer, .tuple = {tuple}, .has_value = value != NULL, .ender = ender, + .has_mark = mark != NULL}; + + if (value != NULL) + std_slice_to_vec_Checked(&take.value, value, value_len, 1, 1); + if (mark != NULL) + std_slice_to_vec_Checked(&take.mark, mark, mark_len, sizeof(uint32), _Alignof(uint32)); + return take; +} + +/* txn_record_tests_versions returns a Vec of carried versions, taking them. */ +static StdVec +txn_record_tests_versions(const TxnRecordCarriedTake *versions, size_t versions_len) +{ + StdVec result; + + std_slice_to_vec_Checked(&result, versions, versions_len, sizeof(TxnRecordCarriedTake), + _Alignof(TxnRecordCarriedTake)); + return result; +} + +/* + * txn_record_tests_version returns a carried version's writer: its op, its + * tuple id when it has one, an entry of [1] when value is set, its ender + * unless it is zero, and its values mark's value ids unless ids is NULL. + */ +static ChunksRecordRecordWriter +txn_record_tests_version(uint64 op, bool has_tuple, uint64 tuple, bool value, uint64 ender, const uint8 *ids, + size_t ids_len) +{ + static const uint8 entry[] = {1}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + chunks_record_RecordWriter_uint(&w, 1, 741); + chunks_record_RecordWriter_uint(&w, 3, 0); + chunks_record_RecordWriter_uint(&w, 4, op); + if (has_tuple) + chunks_record_RecordWriter_uint(&w, 5, tuple); + if (value) + chunks_record_RecordWriter_field(&w, 6, entry, sizeof(entry)); + if (ender != 0) + chunks_record_RecordWriter_uint(&w, 7, ender); + if (ids != NULL) + chunks_record_RecordWriter_field(&w, 8, ids, ids_len); + return w; +} + +/* txn_record_tests_item returns a free space version's writer of tuple id 4. */ +static ChunksRecordRecordWriter +txn_record_tests_item(uint64 op, bool value, uint64 ender) +{ + return txn_record_tests_version(op, true, 4, value, ender, NULL, 0); +} + +/* txn_record_tests_mapped returns a carry's body of one version, a map's number unless it is zero, and an extra field. */ +static StdVec +txn_record_tests_mapped(ChunksRecordRecordWriter item, uint64 map, bool extra) +{ + const TxnRecordTestsField fields[] = {{2, 16500}, {3, 5}, {4, 16384}, {5, 3}}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + StdVec items; + + txn_record_tests_write(&w, fields, lengthof(fields)); + std_slice_to_vec_Checked(&items, &item, 1, sizeof(item), _Alignof(ChunksRecordRecordWriter)); + chunks_record_RecordWriter_list(&w, 6, &items); + if (map != 0) + chunks_record_RecordWriter_uint(&w, 7, map); + if (extra) + chunks_record_RecordWriter_uint(&w, 8, 1); + return chunks_record_RecordWriter_finish(&w); +} + +/* txn_record_tests_carried_refused returns whether a carry's body is refused as data corruption. */ +static bool +txn_record_tests_carried_refused(StdVec body) +{ + return txn_record_tests_carried_read(body) == TXN_RECORD_TESTS_CORRUPT; +} + +/* + * test_txn_record_tests_a_moves_carried_free_space_reads_back_and_refuses_every_other_form: + * PLAN's kind 12 (the rewrite text's section 5): a move's carried free space + * versions read back as written, a subtransaction's writer, an emptying, an + * ended take, and values marks among them, and so do a value map's, and a + * field it doesn't know, an unknown map, a key with no versions, an unknown + * op, an emptying with an entry or an ender, an ender below the first normal + * xid or on a version with no tuple id, a value map version with a tuple id + * or an ender, a values mark with an entry, no tuple id, no value ids, a part + * of one, or in a value map, and value ids on a take are data corruption. + */ +void +test_txn_record_tests_a_moves_carried_free_space_reads_back_and_refuses_every_other_form(void) +{ + static const uint8 taken[] = {9, 8, 7}; + static const uint8 flagged[] = {6}; + static const uint8 five[] = {5}; + static const uint32 mark[] = {16401, 70000}; + static const uint8 id[] = {0x11, 0x40, 0, 0}; + const TxnMemtableVersionWriter sub = {741, true, 740, 2}; + const TxnMemtableVersionWriter emptier = {742, false, 0, 0}; + const TxnMemtableVersionWriter flagger = {743, false, 0, 0}; + const TxnMemtableVersionWriter valuer = {743, false, 0, 1}; + TxnRecordCarriedTake versions[6]; + TxnRecordCarriedTake valued[2]; + TxnRecordMapCarried carried = {.storage = {16500, 5, 16384}, .map = TXN_MEMTABLE_MAP_MAP_FREE, .key = 3}; + TxnRecordMapCarried values = {.storage = {16500, 5, 16384}, .map = TXN_MEMTABLE_MAP_MAP_VALUES, .key = 16400}; + const TxnRecordTestsField empty[] = {{2, 16500}, {3, 5}, {4, 16384}, {5, 3}}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + StdVec none; + + versions[0] = txn_record_tests_take(sub, 3 << 16 | 4, taken, sizeof(taken), 0, NULL, 0); + versions[1] = txn_record_tests_take(sub, 3 << 16 | 4, taken, sizeof(taken), 801, NULL, 0); + versions[2] = txn_record_tests_take(emptier, 3 << 16 | 4, NULL, 0, 0, NULL, 0); + versions[3] = txn_record_tests_take(flagger, 0, flagged, sizeof(flagged), 0, NULL, 0); + versions[4] = txn_record_tests_take(sub, 3 << 16 | 5, NULL, 0, 0, mark, lengthof(mark)); + versions[5] = txn_record_tests_take(sub, 3 << 16 | 5, NULL, 0, 801, mark, lengthof(mark)); + carried.versions = txn_record_tests_versions(versions, lengthof(versions)); + valued[0] = txn_record_tests_take(valuer, 0, five, sizeof(five), 0, NULL, 0); + valued[1] = txn_record_tests_take(valuer, 0, NULL, 0, 0, NULL, 0); + values.versions = txn_record_tests_versions(valued, lengthof(valued)); + + txn_record_tests_carried_round(&carried, __LINE__); + txn_record_MapCarried_drop(&carried); + txn_record_tests_carried_round(&values, __LINE__); + txn_record_MapCarried_drop(&values); + STD_ASSERT_MSG(__LINE__, 9, !txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(1, true, 0), 0, false)), + "a whole body was refused"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(1, true, 0), 0, true)), + "an unknown field decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(4, false, 0), 0, false)), + "an unknown op decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(2, true, 0), 0, false)), + "an emptying with an entry decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(2, false, 801), 0, false)), + "an emptying with an ender decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(1, true, 2), 0, false)), + "an ender below the first normal xid decoded"); + STD_ASSERT_MSG(__LINE__, 9, !txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(1, true, 0), 1, false)), + "a free space map named by its number was refused"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(1, true, 0), 3, false)), + "an unknown map decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_item(1, true, 0), 2, false)), + "a value map version with a tuple id decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(1, false, 0, true, 801, NULL, 0), 0, false)), + "an ender on a free space version with no tuple id decoded"); + STD_ASSERT_MSG(__LINE__, 9, !txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(1, false, 0, true, 0, NULL, 0), 2, false)), + "a value map put was refused"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(1, false, 0, true, 801, NULL, 0), 2, false)), + "a value map version with an ender decoded"); + STD_ASSERT_MSG(__LINE__, 9, !txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(3, true, 4, false, 0, id, sizeof(id)), 0, false)), + "a values mark was refused"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(3, true, 4, true, 0, id, sizeof(id)), 0, false)), + "a values mark with an entry decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(3, true, 0, false, 0, id, sizeof(id)), 0, false)), + "a values mark with no tuple id decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(3, true, 4, false, 0, NULL, 0), 0, false)), + "a values mark with no value ids decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(3, true, 4, false, 0, id, 3), 0, false)), + "a part of a value id decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(3, true, 4, false, 0, id, sizeof(id)), 2, false)), + "a value map's values mark decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(txn_record_tests_mapped(txn_record_tests_version(1, true, 4, true, 0, id, sizeof(id)), 0, false)), + "value ids on a take decoded"); + txn_record_tests_write(&w, empty, lengthof(empty)); + std_vec_new(&none, sizeof(ChunksRecordRecordWriter), _Alignof(ChunksRecordRecordWriter)); + chunks_record_RecordWriter_list(&w, 6, &none); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_carried_refused(chunks_record_RecordWriter_finish(&w)), + "a block with no versions decoded"); +} + +/* txn_record_tests_blob_chunk returns a blob's chunk of "blob" by writer 743, top zero for none. */ +static TxnRecordChunkWrite +txn_record_tests_blob_chunk(uint64 top) +{ + TxnRecordChunkWrite chunk = { + .repository = 16384, + .address = chunks_address_Address_of((const uint8 *) "blob", 4), + .has_writer = true, + .writer = {743, top != 0, top, 0}, + }; + + std_slice_to_vec_Checked(&chunk.bytes, "blob", 4, 1, 1); + return chunk; +} + +/* txn_record_tests_blob_body returns a blob's chunk's body of a repository, an address, bytes, and an extra field. */ +static StdVec +txn_record_tests_blob_body(uint64 repository, const uint8 *address, size_t address_len, const char *bytes, + bool extra) +{ + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + chunks_record_RecordWriter_uint(&w, 1, repository); + chunks_record_RecordWriter_field(&w, 2, address, address_len); + chunks_record_RecordWriter_field(&w, 3, (const uint8 *) bytes, strlen(bytes)); + if (extra) + chunks_record_RecordWriter_uint(&w, 4, 1); + chunks_record_RecordWriter_uint(&w, 10, 743); + return chunks_record_RecordWriter_finish(&w); +} + +/* txn_record_tests_chunk_refused returns whether a chunk's body less its last cut bytes is refused as data corruption. */ +static bool +txn_record_tests_chunk_refused(StdVec body, size_t cut) +{ + return txn_record_tests_chunk_read(body, cut) == TXN_RECORD_TESTS_CORRUPT; +} + +/* + * test_txn_record_tests_a_chunk_reads_back_and_refuses_every_other_form: + * PLAN's Chunk record row (the out-of-line values text, revision 3) and chunk + * spill row: a chunk reads back as it was written, its fields numbered on + * their own with its writer's xids, a subtransaction's top-level one too, and + * a field it doesn't know, a zero or too wide repository, an address of + * another length, bytes whose hash isn't the address, a blob's chunk with no + * writer, a top-level xid with no writer, and a cut body are data corruption. + */ +void +test_txn_record_tests_a_chunk_reads_back_and_refuses_every_other_form(void) +{ + TxnRecordChunkWrite chunk = txn_record_tests_blob_chunk(0); + TxnRecordChunkWrite sub = txn_record_tests_blob_chunk(742); + TxnRecordChunkWrite read = txn_record_tests_chunk_decoded(&chunk, __LINE__); + TxnRecordChunkWrite unwritten = txn_record_tests_blob_chunk(0); + ChunksAddressAddress address = chunks_address_Address_of((const uint8 *) "blob", 4); + const TxnRecordTestsField topped[] = {{1, 16384}, {2, 0, address.f0, sizeof(address.f0)}, {3, 0, "blob", 4}, + {11, 742}}; + StdVec body; + + txn_record_ChunkWrite_drop(&read); + read = txn_record_tests_chunk_decoded(&sub, __LINE__); + txn_record_ChunkWrite_drop(&read); + txn_record_ChunkWrite_drop(&sub); + unwritten.has_writer = false; + unwritten.writer = (TxnMemtableVersionWriter) + { + 0 + }; + txn_record_ChunkWrite_encode(&unwritten, &body); + txn_record_ChunkWrite_drop(&unwritten); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_chunk_refused(body, 0), "a blob's chunk with no writer decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_chunk_refused(txn_record_tests_body(topped, lengthof(topped)), 0), + "a top-level xid with no writer decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_chunk_refused(txn_record_tests_blob_body(16384, address.f0, sizeof(address.f0), + "blob", true), 0), + "an unknown field decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_chunk_refused(txn_record_tests_blob_body(0, address.f0, sizeof(address.f0), + "blob", false), 0), + "a zero repository decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_chunk_refused(txn_record_tests_blob_body(UINT64CONST(1) << 32, address.f0, + sizeof(address.f0), "blob", false), 0), + "a too wide repository decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_chunk_refused(txn_record_tests_blob_body(16384, address.f0, 31, "blob", false), 0), + "a short address decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_chunk_refused(txn_record_tests_blob_body(16384, address.f0, sizeof(address.f0), + "blot", false), 0), + "bytes of another address decoded"); + txn_record_ChunkWrite_encode(&chunk, &body); + txn_record_ChunkWrite_drop(&chunk); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_chunk_refused(body, 1), "a cut chunk decoded"); +} + +/* txn_record_tests_kept_body returns an update's body with one stored value and its kept field unless it is empty. */ +static StdVec +txn_record_tests_kept_body(const uint8 *kept, size_t kept_len) +{ + ChunksAddressAddress ten = chunks_address_Address_of((const uint8 *) "ten", 3); + uint8 stored[1 + sizeof(ten.f0) + 1] = {10}; + const TxnRecordTestsField fields[] = {{1, 41}, {2, 1663}, {3, 5}, {4, 16384}, {6, 0, "k", 1}, {7, 0, "row", 3}, + {9, 2}, {15, 16385}, {16, 3}, {22, 36}, {24, 0, stored, sizeof(stored)}, {25, 1663}, {26, 16390}, {27, 16391}}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + memcpy(stored + 1, ten.f0, sizeof(ten.f0)); + stored[sizeof(stored) - 1] = 1; + txn_record_tests_write(&w, fields, lengthof(fields)); + if (kept_len != 0) + chunks_record_RecordWriter_field(&w, 28, kept, kept_len); + return chunks_record_RecordWriter_finish(&w); +} + +/* + * test_txn_record_tests_an_updates_kept_values_read_back_in_their_values_order: + * field 28 (PLAN's row change record, (4b)): an update that keeps some of its + * row's values out of line reads back with its put's value ids in its values' + * order, field 24 holding the stored ones with their blobs and field 28 the + * kept ones at their places; one that keeps every value has no field 24; and + * a kept place out of order or past the put's values, a kept id also in field + * 24, and an empty field 28 are data corruption. + */ +void +test_txn_record_tests_an_updates_kept_values_read_back_in_their_values_order(void) +{ + static const uint32 updated_ids[] = {10, 11, 12}; + static const uint32 put_ids[] = {20, 21}; + static const size_t one[] = {1}; + static const size_t both[] = {0, 1}; + static const uint8 kept[] = {1, 11}; + static const uint8 past[] = {2, 11}; + static const uint8 stored[] = {1, 10}; + static const uint8 unordered[] = {0, 11, 0, 12}; + const TxnRecordToast toast = {1663, 16390, 16391}; + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_UPDATE_ROW, .u.update_row = {{1}, txn_record_tests_put(1 << 16 | 2)}}; + TxnMemtablePut put = txn_record_tests_put(1 << 16 | 2); + TxnRecordRowChange written; + TxnRecordRowChange all; + + txn_record_tests_ids(&op.u.update_row.f1, updated_ids, lengthof(updated_ids)); + written = txn_record_tests_change(op, 0); + written.has_toast = true; + written.toast = toast; + txn_record_tests_blob(&written, "ten", 3000); + txn_record_tests_blob(&written, "twelve", 5000); + txn_record_tests_kept(&written, one, lengthof(one)); + txn_record_tests_change_round(&written, __LINE__); + txn_record_RowChange_drop(&written); + txn_record_tests_ids(&put, put_ids, lengthof(put_ids)); + all = txn_record_tests_put_change(put, 0); + all.has_toast = true; + all.toast = toast; + txn_record_tests_kept(&all, both, lengthof(both)); + txn_record_tests_change_round(&all, __LINE__); + txn_record_RowChange_drop(&all); + STD_ASSERT(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_kept_body(kept, sizeof(kept)), 0) == + TXN_RECORD_TESTS_READ, "RowChange::decode(&body(&[1, 11])).is_ok()"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_kept_body(past, sizeof(past)), 0) == + TXN_RECORD_TESTS_CORRUPT, "a kept place past the put's values decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_kept_body(stored, sizeof(stored)), 0) == + TXN_RECORD_TESTS_CORRUPT, "a kept id also stored decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_change_read(txn_record_tests_kept_body(unordered, sizeof(unordered)), 0) == + TXN_RECORD_TESTS_CORRUPT, "kept places out of order decoded"); +} + +/* txn_record_tests_pending returns a chain's entry. */ +static TxnBasePending +txn_record_tests_pending(uint64 xid, uint32 command, uint64 sub, size_t transform_len, bool has_end, uint64 end) +{ + TxnBasePending pending = {.xid = xid, .command = command, .sub = sub, .has_end = has_end, .end = end}; + + std_vec_u8_from_elem(&pending.transform, 1, transform_len); + return pending; +} + +/* txn_record_tests_fold_body returns a fold's body of a relation, a base record unless it is NULL, and an extra field. */ +static StdVec +txn_record_tests_fold_body(uint64 relation, const StdVec *kept, size_t kept_cut, bool extra) +{ + static const uint8 pack[16] = {7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}; + const TxnRecordTestsField fields[] = {{2, 1663}, {3, 16384}, {4, relation}, {6, 40}, {7, 742}}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + txn_record_tests_write(&w, fields, lengthof(fields)); + if (kept != NULL) + chunks_record_RecordWriter_field(&w, 8, kept->ptr, kept->len - kept_cut); + chunks_record_RecordWriter_field(&w, 9, pack, sizeof(pack)); + if (extra) + chunks_record_RecordWriter_uint(&w, 11, 1); + return chunks_record_RecordWriter_finish(&w); +} + +/* txn_record_tests_marked_fold returns a fold's whole body with a pack id of a length and one more field. */ +static StdVec +txn_record_tests_marked_fold(const StdVec *kept, size_t pack_len, uint32 number, uint64 value) +{ + static const uint8 pack[16] = {7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}; + const TxnRecordTestsField fields[] = {{2, 1663}, {3, 16384}, {4, 16385}, {6, 40}, {7, 742}, + {8, 0, kept->ptr, kept->len}, {9, 0, pack, pack_len}}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + txn_record_tests_write(&w, fields, lengthof(fields)); + if (number != 0) + chunks_record_RecordWriter_uint(&w, number, value); + return chunks_record_RecordWriter_finish(&w); +} + +/* txn_record_tests_fold_refused returns whether a fold's body less its last cut bytes is refused as data corruption. */ +static bool +txn_record_tests_fold_refused(StdVec body, size_t cut) +{ + return txn_record_tests_advanced_read(body, cut) == TXN_RECORD_TESTS_CORRUPT; +} + +/* + * test_txn_record_tests_a_folds_record_reads_back_and_refuses_every_other_form: + * the fold text's 3.3 and PLAN's fields 10 to 12: a fold's record reads back + * as it was written, its packs' ids, built mark, horizon, and kept mark among + * it, and a field it doesn't know, a too wide storage field, a missing or + * damaged base record, a pack's id cut short, a built mark other than 1, a + * horizon on a fold's record, a kept mark on a fold's record or other than 1, + * and a cut body are data corruption. + */ +void +test_txn_record_tests_a_folds_record_reads_back_and_refuses_every_other_form(void) +{ + TxnRecordBaseAdvanced advanced = { + .storage = {1663, 16384, 16385}, + .folded = UINT64CONST(0x100000028), + .max_folded = 742, + .base = txn_base_tests_full(), + .built = true, + .horizon = 801, + }; + ChunksPackPackId packs[2]; + TxnBasePending entries[2]; + TxnBaseBase full = txn_base_tests_full(); + ChunksRecordRecordWriter w; + ChunksRecordRecordWriter nested = chunks_record_RecordWriter_new(); + StdVec encoded; + StdVec kept; + StdVec chained_value; + + memset(packs[0].f0, 1, sizeof(packs[0].f0)); + memset(packs[1].f0, 2, sizeof(packs[1].f0)); + std_slice_to_vec_Checked(&advanced.packs, packs, lengthof(packs), sizeof(ChunksPackPackId), + _Alignof(ChunksPackPackId)); + txn_record_BaseAdvanced_encode(&advanced, &encoded); + txn_record_tests_advanced_round(&advanced, __LINE__); + entries[0] = txn_record_tests_pending(PG_UINT64_MAX, PG_UINT32_MAX, 7, 9, false, 0); + entries[1] = txn_record_tests_pending(742, 0, 742, 0, true, 801); + advanced.built = false; + advanced.horizon = 0; + advanced.has_chain = true; + std_slice_to_vec_Checked(&advanced.chain, entries, lengthof(entries), sizeof(TxnBasePending), + _Alignof(TxnBasePending)); + txn_record_tests_advanced_round(&advanced, __LINE__); + txn_state_kept(&full, &kept); + txn_base_Base_drop(&full); + w = chunks_record_RecordWriter_new(); + { + const TxnRecordTestsField fields[] = {{2, 1663}, {3, 16384}, {4, 16385}, {6, 40}, {7, 742}, + {8, 0, kept.ptr, kept.len}, {9, 0, "\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7\7", 16}, {10, 1}}; + + txn_record_tests_write(&w, fields, lengthof(fields)); + } + txn_base_chain_value(&STD_VEC_AT(&advanced.chain, TxnBasePending, 0), &chained_value); + chunks_record_RecordWriter_field(&nested, 1, chained_value.ptr, chained_value.len); + std_vec_free_Checked(&chained_value, 1, 1); + chunks_record_RecordWriter_record(&w, 13, &nested); + txn_record_BaseAdvanced_drop(&advanced); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_advanced_read(chunks_record_RecordWriter_finish(&w), 0) != + TXN_RECORD_TESTS_READ, "a chain on a built record decoded"); + STD_ASSERT_MSG(__LINE__, 9, !txn_record_tests_fold_refused(txn_record_tests_fold_body(16385, &kept, 0, false), 0), + "a whole body was refused"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_fold_refused(txn_record_tests_marked_fold(&kept, 16, 10, 2), 0), + "a built mark other than 1 decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_fold_refused(txn_record_tests_marked_fold(&kept, 16, 11, 801), 0), + "a horizon on a fold's record decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_fold_refused(txn_record_tests_fold_body(16385, &kept, 0, true), 0), + "an unknown field decoded"); + STD_ASSERT_MSG(__LINE__, 9, + txn_record_tests_fold_refused(txn_record_tests_fold_body(UINT64CONST(1) << 32, &kept, 0, false), 0), + "a too wide relation decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_fold_refused(txn_record_tests_fold_body(16385, NULL, 0, false), 0), + "a body with no base decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_fold_refused(txn_record_tests_fold_body(16385, &kept, 1, false), 0), + "a damaged base decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_fold_refused(encoded, 1), "a cut body decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_fold_refused(txn_record_tests_marked_fold(&kept, 15, 0, 0), 0), + "a pack's id cut short decoded"); + std_vec_free_Checked(&kept, 1, 1); +} + +/* TxnRecordTestsRuns is a list of runs of tuple ids, block, first offset, count, and bytes. */ +typedef struct TxnRecordTestsRuns +{ + std_TupleU32U16U16U64_t runs[3]; + size_t len; +} TxnRecordTestsRuns; + +/* + * txn_record_tests_freed returns a free of runs and dead runs of a storage + * with no values, each tuple id's stored ender none (bugs row 320). + */ +static TxnRecordTidsFreed +txn_record_tests_freed(const TxnRecordTestsRuns *runs, const TxnRecordTestsRuns *dead) +{ + TxnRecordTidsFreed freed = {.storage = {1663, 5, 16384}}; + uint64 none = 0; + + std_slice_to_vec_Checked(&freed.runs, runs->runs, runs->len, sizeof(std_TupleU32U16U16U64_t), + _Alignof(std_TupleU32U16U16U64_t)); + std_slice_to_vec_Checked(&freed.dead, dead->runs, dead->len, sizeof(std_TupleU32U16U16U64_t), + _Alignof(std_TupleU32U16U16U64_t)); + std_vec_new(&freed.enders, sizeof(uint64), _Alignof(uint64)); + for (size_t i = 0; i < runs->len; i++) + for (uint16 k = 0; k < runs->runs[i].f2; k++) + std_vec_push_Checked(&freed.enders, &none, sizeof(uint64), _Alignof(uint64)); + for (size_t i = 0; i < dead->len; i++) + for (uint16 k = 0; k < dead->runs[i].f2; k++) + std_vec_push_Checked(&freed.enders, &none, sizeof(uint64), _Alignof(uint64)); + return freed; +} + +/* txn_record_tests_freed_status returns what a free's encoding decodes to, freeing the free. */ +static TxnRecordTestsRead +txn_record_tests_freed_status(TxnRecordTidsFreed *freed) +{ + StdVec body; + + txn_record_TidsFreed_encode(freed, &body); + txn_record_TidsFreed_drop(freed); + return txn_record_tests_freed_read(body); +} + +/* txn_record_tests_freed_extra returns a free's encoding with a record of more fields after it, freeing the free. */ +static StdVec +txn_record_tests_freed_extra(TxnRecordTidsFreed *freed, const TxnRecordTestsField *fields, size_t fields_len) +{ + StdVec body; + StdVec extra = txn_record_tests_body(fields, fields_len); + + txn_record_TidsFreed_encode(freed, &body); + txn_record_TidsFreed_drop(freed); + std_vec_extend_from_slice_Checked(&body, extra.ptr, extra.len, 1, 1); + std_vec_free_Checked(&extra, 1, 1); + return body; +} + +/* + * test_txn_record_tests_a_free_reads_back_and_refuses_runs_out_of_bounds_or_order: + * a free's record decodes to what it encoded, its dead runs (D5) too, and a + * run of either field that is empty, starts at offset zero, both (PRevPF 7's + * V1), passes a page's most tuples, isn't past the run before it, or gives + * back more bytes than a page holds, a block whose runs give back more than a + * page, a field 6 with no run, or a field it doesn't know, is data + * corruption. + */ +void +test_txn_record_tests_a_free_reads_back_and_refuses_runs_out_of_bounds_or_order(void) +{ + static const TxnRecordTestsRuns none = {{{0}}, 0}; + static const TxnRecordTestsRuns runs = {{{0, 3, 2, 64}, {0, 291, 1, 28}, {7, 1, 291, 8168}}, 3}; + static const TxnRecordTestsRuns dead_runs = {{{0, 5, 1, 40}, {3, 2, 2, 80}}, 2}; + static const TxnRecordTestsRuns bad_runs[] = { + {{{0, 3, 0, 0}}, 1}, + {{{0, 0, 1, 28}}, 1}, + {{{0, 291, 2, 56}}, 1}, + {{{1, 3, 1, 28}, {1, 2, 1, 28}}, 2}, + {{{1, 3, 2, 56}, {1, 4, 1, 28}}, 2}, + {{{1, 3, 1, 8169}}, 1}, + {{{0, 0, 0, 0}}, 1}, + {{{1, 3, 1, 5000}, {1, 5, 1, 5000}}, 2}, + }; + const TxnRecordTestsField empty[] = {{1, 1663}, {2, 5}, {3, 16384}, {4, 0}, {5, 0, "", 0}, {6, 0, "", 0}}; + const TxnRecordTestsField unknown[] = {{9, 1}}; + TxnRecordTidsFreed freed = txn_record_tests_freed(&runs, &none); + TxnRecordTidsFreed dead = txn_record_tests_freed(&none, &dead_runs); + + txn_record_tests_freed_round(&freed, __LINE__); + txn_record_TidsFreed_drop(&freed); + txn_record_tests_freed_round(&dead, __LINE__); + STD_ASSERT(__LINE__, 9, txn_record_tests_freed_read(txn_record_tests_body(empty, lengthof(empty))) == + TXN_RECORD_TESTS_CORRUPT, "TidsFreed::decode(&empty, 291, 8168).unwrap_err().sqlstate() == b\"XX001\""); + for (size_t i = 0; i < lengthof(bad_runs); i++) + { + TxnRecordTidsFreed bad = txn_record_tests_freed(&bad_runs[i], &none); + + STD_ASSERT(__LINE__, 13, txn_record_tests_freed_status(&bad) == TXN_RECORD_TESTS_CORRUPT, + "TidsFreed::decode(&bad.encode(), 291, 8168).unwrap_err().sqlstate() == b\"XX001\""); + bad = txn_record_tests_freed(&none, &bad_runs[i]); + STD_ASSERT(__LINE__, 13, txn_record_tests_freed_status(&bad) == TXN_RECORD_TESTS_CORRUPT, + "TidsFreed::decode(&bad.encode(), 291, 8168).unwrap_err().sqlstate() == b\"XX001\""); + } + STD_ASSERT(__LINE__, 9, + txn_record_tests_freed_read(txn_record_tests_freed_extra(&dead, unknown, lengthof(unknown))) == + TXN_RECORD_TESTS_CORRUPT, "TidsFreed::decode(&body, 291, 8168).unwrap_err().sqlstate() == b\"XX001\""); +} + +/* txn_record_tests_valued_free returns a free of one run whose values are taken down in a TOAST storage. */ +static TxnRecordTidsFreed +txn_record_tests_valued_free(const std_TupleU32U64_t *values, size_t values_len) +{ + static const TxnRecordTestsRuns none = {{{0}}, 0}; + static const TxnRecordTestsRuns one = {{{0, 3, 1, 64}}, 1}; + TxnRecordTidsFreed freed = txn_record_tests_freed(&one, &none); + + freed.has_values = true; + freed.values.f0 = (HostStorage) + { + 1663, 5, 16387 + }; + std_slice_to_vec_Checked(&freed.values.f1, values, values_len, sizeof(std_TupleU32U64_t), + _Alignof(std_TupleU32U64_t)); + return freed; +} + +/* + * test_txn_record_tests_a_frees_taken_down_values_read_back_and_refuse_every_other_form: + * bugs row 146: a free's taken-down values read back with their TOAST storage + * in the storage's database, and values that are empty, out of id order, or + * of a count of zero, or fields 7 to 9 not all present, are data corruption. + */ +void +test_txn_record_tests_a_frees_taken_down_values_read_back_and_refuse_every_other_form(void) +{ + static const std_TupleU32U64_t values[] = {{7, 1}, {PG_UINT32_MAX, 3}}; + static const std_TupleU32U64_t twice[] = {{7, 1}, {7, 1}}; + static const std_TupleU32U64_t unordered[] = {{9, 1}, {7, 1}}; + static const std_TupleU32U64_t uncounted[] = {{7, 0}}; + static const struct + { + const std_TupleU32U64_t *values; + size_t len; + } bad_values[] = {{values, 0}, {twice, 2}, {unordered, 2}, {uncounted, 1}}; + static const TxnRecordTestsRuns none = {{{0}}, 0}; + static const TxnRecordTestsRuns one = {{{0, 3, 1, 64}}, 1}; + const TxnRecordTestsField partial[] = {{7, 1663}, {8, 16387}}; + TxnRecordTidsFreed freed = txn_record_tests_valued_free(values, lengthof(values)); + TxnRecordTidsFreed plain = txn_record_tests_freed(&one, &none); + + txn_record_tests_freed_round(&freed, __LINE__); + txn_record_TidsFreed_drop(&freed); + for (size_t i = 0; i < lengthof(bad_values); i++) + { + TxnRecordTidsFreed bad = txn_record_tests_valued_free(bad_values[i].values, bad_values[i].len); + + STD_ASSERT(__LINE__, 13, txn_record_tests_freed_status(&bad) == TXN_RECORD_TESTS_CORRUPT, + "TidsFreed::decode(&bad.encode(), 291, 8168).unwrap_err().sqlstate() == b\"XX001\""); + } + STD_ASSERT(__LINE__, 9, + txn_record_tests_freed_read(txn_record_tests_freed_extra(&plain, partial, lengthof(partial))) == + TXN_RECORD_TESTS_CORRUPT, "TidsFreed::decode(&body, 291, 8168).unwrap_err().sqlstate() == b\"XX001\""); +} + +/* TxnRecordTestsExtra is a field an index entry's body carries beside its own, and its integer. */ +typedef struct TxnRecordTestsExtra +{ + uint32 number; + uint64 value; +} TxnRecordTestsExtra; + +/* txn_record_tests_entry_op returns the which'th op of an index entry's change, in the order its test lists them. */ +static TxnMemtableOp +txn_record_tests_entry_op(int which) +{ + TxnMemtableOp op = {0}; + TxnMemtablePut put = txn_record_tests_put(which == 2 ? 3 << 16 | 2 : 0); + + put.heap_bytes = 0; + switch (which) + { + case 0: + op.tag = TXN_MEMTABLE_OP_PUT; + op.u.put = put; + return op; + case 1: + op.tag = TXN_MEMTABLE_OP_DELETE; + break; + case 2: + op.tag = TXN_MEMTABLE_OP_ADD; + op.u.add = put; + return op; + default: + op.tag = TXN_MEMTABLE_OP_REMOVE; + op.u.remove.f0.f0 = 3 << 16 | 2; + break; + } + txn_memtable_Put_drop(&put); + return op; +} + +/* txn_record_tests_entry returns an index entry's body, a put's or a delete's, with extra fields in number order. */ +static StdVec +txn_record_tests_entry(bool put, const TxnRecordTestsExtra *extra, size_t extra_len) +{ + const TxnRecordTestsField fields[] = {{1, 41}, {2, 1663}, {3, 5}, {4, 16390}, {6, 0, "k", 1}}; + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + txn_record_tests_write(&w, fields, lengthof(fields)); + if (put) + chunks_record_RecordWriter_field(&w, 7, (const uint8 *) "entry", 5); + for (size_t i = 0; i < extra_len; i++) + if (extra[i].number < 15) + chunks_record_RecordWriter_uint(&w, extra[i].number, extra[i].value); + chunks_record_RecordWriter_uint(&w, 15, 16385); + chunks_record_RecordWriter_uint(&w, 16, 3); + for (size_t i = 0; i < extra_len; i++) + if (extra[i].number >= 15 && extra[i].number < 31) + chunks_record_RecordWriter_uint(&w, extra[i].number, extra[i].value); + chunks_record_RecordWriter_uint(&w, 31, 16390); + return chunks_record_RecordWriter_finish(&w); +} + +/* txn_record_tests_entry_refused returns whether an index entry's body is refused as data corruption. */ +static bool +txn_record_tests_entry_refused(bool put, const TxnRecordTestsExtra *extra, size_t extra_len) +{ + return txn_record_tests_change_read(txn_record_tests_entry(put, extra, extra_len), 0) == TXN_RECORD_TESTS_CORRUPT; +} + +/* + * test_txn_record_tests_an_index_entrys_change_reads_back_and_refuses_every_field_its_op_doesnt_carry: + * PLAN's row change row, field 31: each change of an index's storage (a put + * naming no tuple id, a delete, and a counted entry's added and removed copy, + * each naming the copy's) reads back as it was written, with a + * subtransaction's xid, field 15 its table's object id and field 31 the + * index's, and no heap bytes; and any field its op doesn't carry beside field + * 31 is data corruption: a put's tuple id, an added copy's removed row, + * another mark, and each of 13, 14, 21, 22, 23, and 24 to 30. + */ +void +test_txn_record_tests_an_index_entrys_change_reads_back_and_refuses_every_field_its_op_doesnt_carry(void) +{ + static const TxnRecordTestsExtra tupled[] = {{9, 1}}; + static const TxnRecordTestsExtra added[] = {{9, 1}, {11, TXN_RECORD_ADD}}; + static const TxnRecordTestsExtra removed[] = {{11, TXN_RECORD_REMOVE}, {12, 1}}; + static const TxnRecordTestsExtra added_removed[] = {{9, 1}, {11, TXN_RECORD_ADD}, {12, 1}}; + static const TxnRecordTestsExtra marked[] = {{11, TXN_RECORD_DELETE_ROW}, {12, 1}}; + static const uint32 numbers[] = {13, 14, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; + + for (int which = 0; which < 4; which++) + { + for (uint64 top = 0; top <= 40; top += 40) + { + TxnRecordRowChange written = txn_record_tests_change(txn_record_tests_entry_op(which), top); + StdVec body; + StdVec fields; + ChunksErrorError failed; + bool carried = true; + + written.heap_bytes = 0; + written.has_index = true; + written.index = 16390; + txn_record_RowChange_encode(&written, &body); + if (!chunks_record_fields(body.ptr, body.len, &fields, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, TXN_RECORD_TESTS_UNWRAP, &failed, chunks_error_Error_Debug_fmt); + for (size_t i = 0; i < fields.len; i++) + { + uint32 number = STD_VEC_AT(&fields, ChunksRecordField, i).number; + + carried = carried && ((number != 9 && number != 22) || + (written.op.tag == TXN_MEMTABLE_OP_ADD && number == 9)); + } + std_vec_free_Checked(&fields, sizeof(ChunksRecordField), _Alignof(ChunksRecordField)); + STD_ASSERT(__LINE__, 17, carried, + "chunks::record::fields(&body).unwrap().iter().all(|field| ![9, 22].contains(&field.number) || matches!(written.op, Op::Add(_)) && field.number == 9)"); + std_vec_free_Checked(&body, 1, 1); + txn_record_tests_change_round(&written, __LINE__); + txn_record_RowChange_drop(&written); + } + } + STD_ASSERT(__LINE__, 9, !txn_record_tests_entry_refused(true, NULL, 0) && + !txn_record_tests_entry_refused(false, NULL, 0), + "!refused(&entry(true, &[])) && !refused(&entry(false, &[]))"); + STD_ASSERT(__LINE__, 9, !txn_record_tests_entry_refused(true, added, lengthof(added)) && + !txn_record_tests_entry_refused(false, removed, lengthof(removed)), + "!refused(&entry(true, &[(9, 1), (11, ADD)])) && !refused(&entry(false, &[(11, REMOVE), (12, 1)]))"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_entry_refused(true, tupled, lengthof(tupled)), + "an entry's put with a tuple id decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_entry_refused(true, added_removed, lengthof(added_removed)), + "an added copy naming a removed row decoded"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_entry_refused(false, marked, lengthof(marked)), + "an entry's change of another mark decoded"); + for (size_t i = 0; i < lengthof(numbers); i++) + { + TxnRecordTestsExtra extra = {numbers[i], 1}; + + txn_record_tests_message_decoded(txn_record_tests_entry_refused(true, &extra, 1), + "an entry's put with field ", &numbers[i], std_u32_Debug_fmt, __LINE__, 13); + txn_record_tests_message_decoded(txn_record_tests_entry_refused(false, &extra, 1), + "an entry's delete with field ", &numbers[i], std_u32_Debug_fmt, __LINE__, + 13); + } +} + +/* txn_record_tests_key returns the key of a chunk's first piece, taking the chunk. */ +static StdVec +txn_record_tests_key(TxnRecordChunkWrite chunk) +{ + StdVec pieces; + StdVec key; + HostError error; + + if (!txn_record_ChunkWrite_pieces(&chunk, &pieces, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, TXN_RECORD_TESTS_UNWRAP, &error, host_Error_Debug_fmt); + std_vec_clone(&key, &STD_VEC_AT(&pieces, txn_TupleVecU8VecU8_t, 0).f0, 1, 1); + txn_record_tests_pieces_drop(&pieces); + return key; +} + +/* txn_record_tests_keys_ne asserts at a line that two keys differ, freeing the second. */ +static void +txn_record_tests_keys_ne(const StdVec *left, StdVec right, const char *message, uint32 line) +{ + if (txn_record_tests_bytes_eq(left, right.ptr, right.len)) + { + StdFmtPiece pieces[] = {{message, strlen(message), 0, {0}}}; + StdFmtArguments arguments = {pieces, 1, NULL}; + + std_assert_failed(RUST_FILE, line, 9, "!=", left, txn_record_tests_vec_Debug_fmt, &right, + txn_record_tests_vec_Debug_fmt, &arguments); + } + std_vec_free_Checked(&right, 1, 1); +} + +/* txn_record_tests_node_refused returns whether a node's chunk of the numbered fields beside its own is refused. */ +static bool +txn_record_tests_node_refused(const StdVec *leaf, const StdVec *stamps, const uint32 *numbers, size_t numbers_len) +{ + ChunksAddressAddress address = chunks_address_Address_of(leaf->ptr, leaf->len); + ChunksRecordRecordWriter w = chunks_record_RecordWriter_new(); + + chunks_record_RecordWriter_uint(&w, 1, 16384); + chunks_record_RecordWriter_field(&w, 2, address.f0, sizeof(address.f0)); + chunks_record_RecordWriter_field(&w, 3, leaf->ptr, leaf->len); + for (size_t i = 0; i < numbers_len; i++) + { + if (numbers[i] == 5) + chunks_record_RecordWriter_field(&w, 5, stamps->ptr, stamps->len); + else + chunks_record_RecordWriter_uint(&w, numbers[i], 1); + } + return txn_record_tests_chunk_refused(chunks_record_RecordWriter_finish(&w), 0); +} + +/* + * test_txn_record_tests_a_built_nodes_key_names_its_base_and_its_fields_come_together: + * PLAN's Chunk record row, fields 6 to 9 (CommPF 8's ruling on concurrent + * built bases): a built base's node is keyed in the working state's delta by + * its address and its base, so two bases' nodes of one address are held + * apart and a blob's chunk keeps its address alone; and stamps without a + * storage, a fold point, and a writer, or any of those without stamps, are + * data corruption. + */ +void +test_txn_record_tests_a_built_nodes_key_names_its_base_and_its_fields_come_together(void) +{ + static const uint32 whole[] = {5, 6, 7, 8, 9}; + static const uint32 written[] = {5, 6, 7, 8, 9, 10}; + static const uint32 first_set[] = {5}; + static const uint32 second_set[] = {5, 6, 7}; + static const uint32 third_set[] = {5, 6, 7, 8}; + static const uint32 fourth_set[] = {6, 7, 8, 9}; + static const uint32 fifth_set[] = {9}; + static const TxnRecordTestsU32s refused_sets[] = {{first_set, 1}, {second_set, 3}, {third_set, 4}, + {fourth_set, 4}, {fifth_set, 1}}; + StdVec leaf = txn_record_tests_leaf(1); + ChunksTrailerStamps one = txn_record_tests_stamps(1); + ChunksAddressAddress address = chunks_address_Address_of(leaf.ptr, leaf.len); + TxnRecordChunkWrite alone = txn_record_tests_node(&leaf, &one, 16390, 0); + StdVec first = txn_record_tests_key(txn_record_tests_node(&leaf, &one, 16390, UINT64CONST(0x100000028))); + StdVec stamps; + StdVec alone_key; + StdBytes head = {first.ptr, Min(first.len, TXN_MEMTABLE_BRANCH_ADDRESS)}; + StdBytes addressed = {address.f0, sizeof(address.f0)}; + + STD_ASSERT_EQ_U64(__LINE__, 9, first.len, TXN_MEMTABLE_BRANCH_PIECE_KEY); + STD_ASSERT_EQ(__LINE__, 9, head.len == addressed.len && memcmp(head.ptr, addressed.ptr, head.len) == 0, &head, + std_bytes_Debug_fmt, &addressed, std_bytes_Debug_fmt); + txn_record_tests_keys_ne(&first, txn_record_tests_key(txn_record_tests_node(&leaf, &one, 16391, + UINT64CONST(0x100000028))), + "two storages' nodes share a key", __LINE__); + txn_record_tests_keys_ne(&first, txn_record_tests_key(txn_record_tests_node(&leaf, &one, 16390, + UINT64CONST(0x100000030))), + "two fold points' nodes share a key", __LINE__); + std_vec_free_Checked(&alone.node.stamps, 1, 1); + alone.has_node = false; + alone.node = (TxnRecordBuiltNode) + { + 0 + }; + alone_key = txn_record_tests_key(alone); + STD_ASSERT_EQ(__LINE__, 9, txn_record_tests_bytes_eq(&alone_key, address.f0, sizeof(address.f0)), &alone_key, + txn_record_tests_vec_Debug_fmt, &addressed, std_bytes_Debug_fmt); + std_vec_free_Checked(&alone_key, 1, 1); + chunks_trailer_store((const uint8 *) "", 0, NULL, 0, &one, &stamps); + STD_ASSERT_MSG(__LINE__, 9, !txn_record_tests_node_refused(&leaf, &stamps, whole, lengthof(whole)), + "a built node's whole fields were refused"); + STD_ASSERT_MSG(__LINE__, 9, txn_record_tests_node_refused(&leaf, &stamps, written, lengthof(written)), + "a built node with a blob's writer decoded"); + for (size_t i = 0; i < lengthof(refused_sets); i++) + txn_record_tests_fields_decoded(txn_record_tests_node_refused(&leaf, &stamps, refused_sets[i].ptr, + refused_sets[i].len), + refused_sets[i].ptr, refused_sets[i].len, __LINE__); + std_vec_free_Checked(&stamps, 1, 1); + std_vec_free_Checked(&first, 1, 1); + chunks_trailer_Stamps_drop(&one); + std_vec_free_Checked(&leaf, 1, 1); +} + +#endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/rewrite.c b/src/doltgres/txn/c/rewrite.c index 7bf2086a7e0da..9b82aaea91ab6 100644 --- a/src/doltgres/txn/c/rewrite.c +++ b/src/doltgres/txn/c/rewrite.c @@ -39,7 +39,11 @@ #include "doltgres/txn/memtable/table.h" #include "doltgres/txn/memtable/tuples.h" #include "doltgres/txn/rewrite.h" +#include "doltgres/txn/spill.h" +#include "doltgres/txn/state.h" +#include "doltgres/txn/status.h" #include "doltgres/txn/visible.h" +#include "doltgres/txn/written.h" #include "port/pg_bswap.h" #define RUST_FILE "txn" STD_SEP "src" STD_SEP "rewrite.rs" @@ -292,6 +296,9 @@ typedef struct txn_rewrite_keys_frame_t HostDeltaHold hold; bool versions_live; StdVec versions; + /* after is the key the walk handed last, while has_after. */ + bool has_after; + StdVec after; HostError *error; bool ok; } txn_rewrite_keys_frame_t; @@ -334,35 +341,7 @@ txn_rewrite_place(const txn_rewrite_keys_frame_t *frame, const uint8 *key, size_ return false; } -/* txn_rewrite_keys_closure_1 is keys' walk of the delta's rows: each key with its place and its unfolded versions. */ -static bool -txn_rewrite_keys_closure_1(void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *values, - size_t values_len, HostError *error) -{ - txn_rewrite_keys_frame_t *frame = captures; - HostSharedRecord record = host_shared_Held_record(frame->held); - size_t place = 0; - bool has_place; - - if (!host_delta_hold(io, &frame->hold, error)) - return false; - frame->hold_live = true; - if (!txn_visible_versions(io, &frame->hold, record, key, key_len, frame->folded, &frame->versions, error)) - return false; - frame->versions_live = true; - has_place = txn_rewrite_place(frame, key, key_len, &place); - if (has_place) - STD_VEC_AT(&frame->handed, bool, place) = true; - if (!frame->each(frame->each_captures, io, key, key_len, has_place, place, &frame->versions, error)) - return false; - frame->versions_live = false; - txn_memtable_version_versions_drop(&frame->versions, 0); - frame->hold_live = false; - host_delta_Hold_drop(&frame->hold); - return true; -} - -/* txn_rewrite_keys_body walks the delta's keys, then hands each base entry it didn't hand with no versions. */ +/* txn_rewrite_keys_body walks the delta's keys, its memtable's and its spills' merged, then hands each base entry it didn't hand with no versions. */ static void txn_rewrite_keys_body(void *frame_) { @@ -380,9 +359,43 @@ txn_rewrite_keys_body(void *frame_) frame->handed_live = true; for (size_t at = 0; at < frame->based_len; at++) std_vec_push_Checked(&frame->handed, &unhanded, sizeof(bool), _Alignof(bool)); - if (!txn_memtable_table_walked(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, frame->io, host_shared_Held_record(frame->held), - txn_rewrite_keys_closure_1, frame, NULL, frame->error)) - return; + for (;;) + { + txn_TupleVecU8VecVersion_t next; + bool some; + size_t place = 0; + bool has_place; + + if (!host_delta_hold(frame->io, &frame->hold, frame->error)) + return; + frame->hold_live = true; + if (!txn_visible_next(frame->io, &frame->hold, host_shared_Held_record(frame->held), frame->has_after, + frame->after.ptr, frame->after.len, TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, frame->folded, + &some, &next, frame->error)) + return; + if (!some) + { + frame->hold_live = false; + host_delta_Hold_drop(&frame->hold); + break; + } + if (frame->has_after) + std_vec_free_Checked(&frame->after, 1, 1); + frame->after = next.f0; + frame->has_after = true; + frame->versions = next.f1; + frame->versions_live = true; + has_place = txn_rewrite_place(frame, frame->after.ptr, frame->after.len, &place); + if (has_place) + STD_VEC_AT(&frame->handed, bool, place) = true; + if (!frame->each(frame->each_captures, frame->io, frame->after.ptr, frame->after.len, has_place, place, + &frame->versions, frame->error)) + return; + frame->versions_live = false; + txn_memtable_version_versions_drop(&frame->versions, 0); + frame->hold_live = false; + host_delta_Hold_drop(&frame->hold); + } for (size_t at = 0; at < frame->based_len; at++) { if (STD_VEC_AT(&frame->handed, bool, at)) @@ -406,6 +419,9 @@ txn_rewrite_keys_pad(void *frame_) if (frame->versions_live) txn_memtable_version_versions_drop(&frame->versions, 0); + if (frame->has_after) + std_vec_free_Checked(&frame->after, 1, 1); + frame->has_after = false; if (frame->hold_live) host_delta_Hold_drop(&frame->hold); if (frame->handed_live) @@ -421,8 +437,9 @@ txn_rewrite_keys_pad(void *frame_) * txn_rewrite_keys hands each every key a base's entries or a storage's delta * holds, once, with the place of its entry among the base's and its versions * newest first, less those the base folded at folded applied: the delta's - * keys in their bytes' order, then the base's other entries in its key order - * with none. + * keys in their bytes' order, its memtable's and its spills' merged, so a key + * whose versions a spill took out of the memtable is carried too, then the + * base's other entries in its key order with none. */ bool txn_rewrite_keys(HostIoIo *io, const HostSharedHeld *held, const TxnRewriteBased *based, size_t based_len, @@ -704,6 +721,7 @@ txn_rewrite_new_base_body(void *frame_) *frame->result = *frame->made; frame->made_live = false; frame->result->folded = frame->folded; + frame->result->read = frame->folded; frame->result->max_folded = frame->max_folded; frame->result->max_ended = frame->max_ended; frame->result->counter = frame->packed.f0; @@ -796,6 +814,12 @@ typedef struct txn_rewrite_carried_closure_5_captures_t size_t marking; TxnRecordEncoding *encoding; bool *encoding_live; + + /* + * moved is whether the place's take found its block's entry moved since + * its read, pushing nothing. + */ + bool moved; } txn_rewrite_carried_closure_5_captures_t; /* @@ -833,10 +857,18 @@ txn_rewrite_carried_closure_5_body(void *frame_) if (!host_delay_checkpoint(HOST_IO_IO_IO_DOORS(frame->io), NULL, 0, &frame->window, frame->error)) return; frame->window_live = true; + c->moved = false; if (c->has_placed && - (!txn_insert_place_taken(io, frame->prepared, 0, record, c->change->writer, c->placed, frame->error) || - !txn_memtable_table_counted(io, frame->prepared, c->has_roomed, c->roomed, frame->error))) + !txn_insert_place_taken(io, frame->prepared, 0, record, c->change->writer, c->has_roomed, c->roomed, + c->placed, &c->moved, frame->error)) + return; + if (c->moved) + { + frame->window_live = false; + host_DelayCheckpoint_finish(&frame->window); + frame->ok = true; return; + } if (c->logged) { *c->encoding_live = false; @@ -855,9 +887,8 @@ txn_rewrite_carried_closure_5_body(void *frame_) c->has_marks = false; std_vec_new(c->marks, TXN_REWRITE_ID); - if (!txn_memtable_map_named(io, frame->prepared, c->marking, c->change->writer, pushed.has_lsn, pushed.lsn, - c->tuple, &marks, frame->error) || - !txn_memtable_table_counted(io, frame->prepared, c->has_roomed, c->roomed, frame->error)) + if (!txn_memtable_map_named(io, frame->prepared, c->marking, c->change->writer, c->has_roomed, c->roomed, + pushed.has_lsn, pushed.lsn, c->tuple, &marks, frame->error)) return; } if (!txn_memtable_table_finish(io, frame->prepared, &pushed, c->has_roomed, c->roomed, &handle, frame->error)) @@ -914,6 +945,7 @@ typedef struct txn_rewrite_carried_frame_t StdVec marks; bool encoding_live; TxnRecordEncoding encoding; + TxnMemtableTableRefusal refusal; HostError *error; bool ok; } txn_rewrite_carried_frame_t; @@ -968,10 +1000,60 @@ txn_rewrite_carried_body(void *frame_) row.f0.ptr = change->key.ptr; row.f0.len = change->key.len; row.f1 = txn_memtable_version_encoded(&change->writer, &change->op, frame->shown, captures.logged, 0); + if (!locks) + { + uint64 top = change->writer.has_top ? change->writer.top : change->writer.xid; + uint64 touched; + bool rows; + bool takes_over = false; + bool spilled; + + if (!txn_written_touched(frame->io, top, &change->storage, TXN_WRITTEN_OF_ROWS, change->key.ptr, + change->key.len, row.f1, &touched, frame->error)) + return; + rows = touched >= txn_spill_threshold(); + for (size_t k = 0; k < takes_len; k++) + { + uint8 block[4]; + + std_u32_to_be_bytes(takes[k].key, block); + if (!txn_written_touched(frame->io, top, &change->storage, TXN_WRITTEN_OF_FREE, block, sizeof(block), + takes[k].most, &touched, frame->error)) + return; + takes_over |= touched >= txn_spill_threshold(); + } + if ((rows && !txn_spill_spilled(frame->io, &change->storage, TXN_WRITTEN_OF_ROWS, top, &spilled, + frame->error)) || + (takes_over && !txn_spill_spilled(frame->io, &change->storage, TXN_WRITTEN_OF_FREE, top, &spilled, + frame->error))) + return; + } txn_record_RowChange_encoding(change, &frame->encoding); frame->encoding_live = true; - frame->ok = txn_memtable_table_prepare_Unit(frame->io, record, &row, 1, takes, takes_len, captures.has_tuple, - txn_rewrite_carried_closure_5, &captures, NULL, frame->error); + for (;;) + { + if (!txn_memtable_table_readied(frame->io, true, record, &row, 1, takes, takes_len, captures.has_tuple, + txn_rewrite_carried_closure_5, &captures, NULL, &frame->refusal, + frame->error)) + return; + if (!frame->refusal.refused && !captures.moved) + { + frame->ok = true; + return; + } + if (frame->refusal.refused && !txn_memtable_table_waited(frame->io, frame->refusal.full, false, frame->error)) + return; + if (frame->placed_live) + { + frame->placed_live = false; + txn_insert_Placed_drop(&frame->placed); + if (!txn_insert_Placed_read(frame->io, frame->store, frame->held, put->tuple, put->heap_bytes, + frame->page, &frame->placed, frame->error)) + return; + frame->placed_live = true; + takes[0] = txn_insert_Placed_take(&frame->placed, record); + } + } } /* txn_rewrite_carried_pad frees what carried holds live. */ @@ -990,20 +1072,24 @@ txn_rewrite_carried_pad(void *frame_) } /* - * txn_rewrite_carried pushes a version a rewrite carries into its new - * storage: in one checkpoint window, a logged storage's row change record - * under the version's own writer, whose end becomes the version's LSN, then - * the version with the command it showed and the row lock it held, counted - * at its writer's top-level entry, and a table row's tuple id in the tuple id - * table; a lock is never logged and counts nothing. Given the page a rewrite - * placed it on, a table row's put first takes its tuple id's place as the - * writer's, counted at the writer's entry, and one that names values out of - * line pushes its values mark after its record. + * txn_rewrite_own_carried pushes a version of the carrying transaction's own + * into its new storage: in one checkpoint window, a logged storage's row + * change record under the version's own writer, whose end becomes the + * version's LSN, then the version with the command it showed and the row + * lock it held, counted at its writer's top-level entry, and a table row's + * tuple id in the tuple id table; a lock is never logged and counts nothing. + * Given the page a rewrite placed it on, a table row's put first takes its + * tuple id's place as the writer's, counted at the writer's entry, and one + * that names values out of line pushes its values mark after its record. It + * counts toward the transaction's own spill, row and take, as an insert's + * does, so a rewrite of its own rows past the threshold spills them rather + * than waiting at the budget (bugs row 229), and a refusal all the same waits + * at the budget, reading its place again. */ -bool -txn_rewrite_carried(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, - const TxnRecordRowChange *change, TxnMemtableVersionShown shown, TxnMemtableLockMode lock, - bool logged, const TxnInsertPage *page, HostError *error) +static bool +txn_rewrite_own_carried(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, + const TxnRecordRowChange *change, TxnMemtableVersionShown shown, TxnMemtableLockMode lock, + bool logged, const TxnInsertPage *page, HostError *error) { txn_rewrite_carried_frame_t frame = {.io = io, .store = store, .held = held, .change = change, .shown = shown, .lock = lock, .logged = logged, .page = page, .error = error}; @@ -1013,6 +1099,463 @@ txn_rewrite_carried(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHel return frame.ok; } +/* txn_rewrite_top returns a writer's top-level xid. */ +static uint64 +txn_rewrite_top(const TxnMemtableVersionWriter *writer) +{ + return writer->has_top ? writer->top : writer->xid; +} + +/* + * txn_rewrite_chained adds a version, which it takes, to the chain of a + * tree's key a carry spills, after the versions it holds, starting a chain + * for a key it holds none of; the key is copied into the chain it starts, + * which owns it. + */ +static bool +txn_rewrite_chained(StdVec *chains, TxnWrittenOf of, const uint8 *key, size_t key_len, uint64 xid, StdVec record, + HostError *error) +{ + TxnSpillChain *chain = NULL; + txn_TupleU64VecU8_t version = {xid, record}; + + for (size_t k = chains->len; k > 0 && chain == NULL; k--) + { + TxnSpillChain *at = &STD_VEC_AT(chains, TxnSpillChain, k - 1); + + if (at->of == of && at->key.len == key_len && memcmp(at->key.ptr, key, key_len) == 0) + chain = at; + } + if (chain == NULL) + { + TxnSpillChain made = {.of = of}; + + if (!std_vec_try_reserve(chains, 1, sizeof(made), _Alignof(TxnSpillChain))) + { + std_vec_free_Checked(&record, 1, 1); + *error = host_Error_out_of_memory(); + return false; + } + std_slice_to_vec_Checked(&made.key, key, key_len, 1, 1); + std_vec_new(&made.versions, sizeof(txn_TupleU64VecU8_t), _Alignof(txn_TupleU64VecU8_t)); + std_vec_push_Checked(chains, &made, sizeof(made), _Alignof(TxnSpillChain)); + chain = &STD_VEC_AT(chains, TxnSpillChain, chains->len - 1); + } + if (!std_vec_try_reserve(&chain->versions, 1, sizeof(version), _Alignof(txn_TupleU64VecU8_t))) + { + std_vec_free_Checked(&record, 1, 1); + *error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(&chain->versions, &version, sizeof(version), _Alignof(txn_TupleU64VecU8_t)); + return true; +} + +/* + * TxnRewriteTheirs is carried's spilled part: each map key's versions below + * its own run, as a view of its MapCarried, the other writers' carries, the + * puts among them and their takes, and whether the storage is logged. + */ +typedef struct TxnRewriteTheirs +{ + /* + * spaces is a Vec, each a view of its versions below its own + * run. + */ + StdVec spaces; + /* carries is a Vec<&Carry>. */ + StdVec carries; + /* puts is a Vec, and takes the Vec of them. */ + StdVec puts; + StdVec takes; + bool logged; +} TxnRewriteTheirs; + +/* txn_rewrite_theirs_closure_0 is carried's logging under the spill files' hold: each spilled key's map versions as one record, each spilled row version's own, and the takes, as chains. */ +static bool +txn_rewrite_theirs_closure_0(void *captures, HostSharedLongIo *io, StdVec *chains, HostError *error) +{ + TxnRewriteTheirs *c = captures; + HostIoIoDoors doors = HOST_SHARED_LONG_IO_IO_DOORS(io); + TxnMemtableVersionShown command = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + + for (size_t s = 0; s < c->spaces.len; s++) + { + const TxnRecordMapCarried *space = &STD_VEC_AT(&c->spaces, TxnRecordMapCarried, s); + TxnWrittenOf of = space->map == TXN_MEMTABLE_MAP_MAP_FREE ? TXN_WRITTEN_OF_FREE : TXN_WRITTEN_OF_VALUES; + bool has_lsn = c->logged; + uint64 lsn = 0; + uint8 key[4]; + + if (c->logged) + { + StdVec body; + bool ok; + + txn_record_MapCarried_encode(space, &body); + ok = host_wal_insert(doors, HOST_WAL_KIND_MAP_CARRIED, body.ptr, body.len, &lsn, error); + std_vec_free_Checked(&body, 1, 1); + if (!ok) + return false; + } + std_u32_to_be_bytes(space->key, key); + for (size_t v = 0; v < space->versions.len; v++) + { + const TxnRecordCarriedTake *version = &STD_VEC_AT(&space->versions, TxnRecordCarriedTake, v); + TxnMemtableOp op = {0}; + StdVec record; + + if (version->has_mark) + { + op.tag = TXN_MEMTABLE_OP_NAMED; + op.u.named.f0 = version->tuple; + op.u.named.f1 = version->mark; + } + else if (version->has_value) + { + op.tag = TXN_MEMTABLE_OP_PUT; + op.u.put.value = version->value; + op.u.put.tuple = version->tuple; + } + else if (version->tuple.f0 == 0) + op.tag = TXN_MEMTABLE_OP_DELETE; + else + { + op.tag = TXN_MEMTABLE_OP_EMPTIED; + op.u.emptied = version->tuple; + } + std_vec_new(&record, 1, 1); + txn_memtable_version_encode(&version->writer, &op, TXN_MEMTABLE_LOCK_MODE_UPDATE, command, has_lsn, lsn, + &record); + if (version->ender != 0) + txn_memtable_version_ended(&record, (uint32) version->ender); + if (!txn_rewrite_chained(chains, of, key, sizeof(key), txn_rewrite_top(&version->writer), record, error)) + return false; + } + } + for (size_t k = 0; k < c->carries.len; k++) + { + const TxnRewriteCarry *carry = STD_VEC_AT(&c->carries, const TxnRewriteCarry *, k); + const TxnRecordRowChange *change = &carry->f0; + bool locks = change->op.tag == TXN_MEMTABLE_OP_LOCK || change->op.tag == TXN_MEMTABLE_OP_LOCK_ROW; + bool has_lsn = c->logged && !locks; + uint64 lsn = 0; + StdVec record; + + if (has_lsn) + { + TxnRecordEncoding encoding; + StdVec body; + bool ok; + + txn_record_RowChange_encoding(change, &encoding); + txn_record_RowChange_encode_with(change, &encoding, &body); + ok = host_wal_insert(doors, HOST_WAL_KIND_ROW_CHANGE, body.ptr, body.len, &lsn, error); + std_vec_free_Checked(&body, 1, 1); + if (!ok) + return false; + } + std_vec_new(&record, 1, 1); + txn_memtable_version_encode(&change->writer, &change->op, carry->f2, carry->f1, has_lsn, lsn, &record); + if (!txn_rewrite_chained(chains, TXN_WRITTEN_OF_ROWS, change->key.ptr, change->key.len, + txn_rewrite_top(&change->writer), record, error)) + return false; + } + for (size_t k = 0; k < c->takes.len; k++) + { + TxnInsertCarriedTake *take = &STD_VEC_AT(&c->takes, TxnInsertCarriedTake, k); + const TxnInsertCarriedPut *put = &STD_VEC_AT(&c->puts, TxnInsertCarriedPut, k); + uint8 block[4]; + StdVec record = take->record; + + std_vec_new(&take->record, 1, 1); + std_u32_to_be_bytes(take->block, block); + if (!txn_rewrite_chained(chains, TXN_WRITTEN_OF_FREE, block, sizeof(block), txn_rewrite_top(&put->writer), + record, error)) + return false; + } + return true; +} + +/* txn_rewrite_carried_frame_1_t is carried's state across its pad: the own runs' views and carries, and the spilled part, while live. */ +typedef struct txn_rewrite_carried_frame_1_t +{ + HostIoIo *io; + TreesStoreNodeStore store; + const HostSharedHeld *held; + const TxnRecordMapCarried *spaces; + size_t spaces_len; + const TxnRewriteCarry *carries; + size_t carries_len; + bool logged; + const TxnInsertPage *page; + uint16 most; + HostError *error; + bool ok; + /* own_spaces is a Vec of views, and own a Vec<&Carry>. */ + StdVec own_spaces; + StdVec own; + TxnRewriteTheirs theirs; + bool live; + + /* + * writers is a Vec: the top-level xids the carry of others' versions + * may spill under. + */ + StdVec writers; +} txn_rewrite_carried_frame_1_t; + +/* txn_rewrite_carried_pad_1 is carried's pad: its vectors, while live. */ +static void +txn_rewrite_carried_pad_1(void *frame_) +{ + txn_rewrite_carried_frame_1_t *frame = frame_; + + if (!frame->live) + return; + frame->live = false; + std_vec_free_Checked(&frame->own_spaces, sizeof(TxnRecordMapCarried), _Alignof(TxnRecordMapCarried)); + std_vec_free_Checked(&frame->own, sizeof(const TxnRewriteCarry *), _Alignof(const TxnRewriteCarry *)); + std_vec_free_Checked(&frame->theirs.spaces, sizeof(TxnRecordMapCarried), _Alignof(TxnRecordMapCarried)); + std_vec_free_Checked(&frame->theirs.carries, sizeof(const TxnRewriteCarry *), _Alignof(const TxnRewriteCarry *)); + std_vec_free_Checked(&frame->theirs.puts, sizeof(TxnInsertCarriedPut), _Alignof(TxnInsertCarriedPut)); + txn_insert_carried_takes_drop(&frame->theirs.takes); + std_vec_free_Checked(&frame->writers, sizeof(uint64), _Alignof(uint64)); +} + +/* txn_rewrite_carried_push adds an element to one of carried's vectors, refusing it when the vector can't grow. */ +static bool +txn_rewrite_carried_push(txn_rewrite_carried_frame_1_t *frame, StdVec *vector, const void *elem, size_t size, + size_t align) +{ + if (!std_vec_try_reserve(vector, 1, size, align)) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(vector, elem, size, align); + return true; +} + +/* txn_rewrite_current reports whether a writer's top-level transaction is the carrying one. */ +static bool +txn_rewrite_current(HostIoIo *io, const TxnMemtableVersionWriter *writer) +{ + HostXid xid = {txn_rewrite_top(writer)}; + + return host_is_current(HOST_IO_IO_IO_DOORS(io), xid); +} + +/* txn_rewrite_writer adds a top-level xid to the carry's writers once, leaving out one below the first normal xid. */ +static bool +txn_rewrite_writer(txn_rewrite_carried_frame_1_t *frame, uint64 top) +{ + for (size_t k = 0; k < frame->writers.len; k++) + if (STD_VEC_AT(&frame->writers, uint64, k) == top) + return true; + return top < TXN_STATUS_FIRST_NORMAL || + txn_rewrite_carried_push(frame, &frame->writers, &top, sizeof(uint64), _Alignof(uint64)); +} + +/* txn_rewrite_writers gathers the top-level xids of the other writers' versions the carry may spill under. */ +static bool +txn_rewrite_writers(txn_rewrite_carried_frame_1_t *frame) +{ + for (size_t s = 0; s < frame->theirs.spaces.len; s++) + { + const TxnRecordMapCarried *space = &STD_VEC_AT(&frame->theirs.spaces, TxnRecordMapCarried, s); + + for (size_t v = 0; v < space->versions.len; v++) + if (!txn_rewrite_writer(frame, txn_rewrite_top(&STD_VEC_AT(&space->versions, TxnRecordCarriedTake, + v).writer))) + return false; + } + for (size_t k = 0; k < frame->theirs.carries.len; k++) + if (!txn_rewrite_writer(frame, txn_rewrite_top(&STD_VEC_AT(&frame->theirs.carries, const TxnRewriteCarry *, + k)->f0.writer))) + return false; + for (size_t k = 0; k < frame->theirs.puts.len; k++) + if (!txn_rewrite_writer(frame, txn_rewrite_top(&STD_VEC_AT(&frame->theirs.puts, TxnInsertCarriedPut, k).writer))) + return false; + return true; +} + +/* txn_rewrite_carried_body_1 is carried's body. */ +static void +txn_rewrite_carried_body_1(void *frame_) +{ + txn_rewrite_carried_frame_1_t *frame = frame_; + + std_vec_new(&frame->own_spaces, sizeof(TxnRecordMapCarried), _Alignof(TxnRecordMapCarried)); + std_vec_new(&frame->own, sizeof(const TxnRewriteCarry *), _Alignof(const TxnRewriteCarry *)); + std_vec_new(&frame->theirs.spaces, sizeof(TxnRecordMapCarried), _Alignof(TxnRecordMapCarried)); + std_vec_new(&frame->theirs.carries, sizeof(const TxnRewriteCarry *), _Alignof(const TxnRewriteCarry *)); + std_vec_new(&frame->theirs.puts, sizeof(TxnInsertCarriedPut), _Alignof(TxnInsertCarriedPut)); + std_vec_new(&frame->writers, sizeof(uint64), _Alignof(uint64)); + std_vec_new(&frame->theirs.takes, sizeof(TxnInsertCarriedTake), _Alignof(TxnInsertCarriedTake)); + frame->theirs.logged = frame->logged; + frame->live = true; + for (size_t s = 0; s < frame->spaces_len; s++) + { + const TxnRecordMapCarried *space = &frame->spaces[s]; + size_t split = space->versions.len; + TxnRecordMapCarried view = *space; + + while (split > 0 && txn_rewrite_current(frame->io, &STD_VEC_AT(&space->versions, TxnRecordCarriedTake, + split - 1).writer)) + split--; + if (split < space->versions.len) + { + view.versions = (StdVec) + { + .ptr = (uint8 *) space->versions.ptr + split * sizeof(TxnRecordCarriedTake), + .len = space->versions.len - split, .cap = space->versions.len - split + }; + if (!txn_rewrite_carried_push(frame, &frame->own_spaces, &view, sizeof(view), + _Alignof(TxnRecordMapCarried))) + return; + } + if (split > 0) + { + view.versions = (StdVec) + { + .ptr = space->versions.ptr, .len = split, .cap = split + }; + if (!txn_rewrite_carried_push(frame, &frame->theirs.spaces, &view, sizeof(view), + _Alignof(TxnRecordMapCarried))) + return; + } + } + for (size_t start = 0; start < frame->carries_len;) + { + const StdVec *key = &frame->carries[start].f0.key; + size_t end = start; + size_t split; + + while (end < frame->carries_len && frame->carries[end].f0.key.len == key->len && + memcmp(frame->carries[end].f0.key.ptr, key->ptr, key->len) == 0) + end++; + split = end; + while (split > start && txn_rewrite_current(frame->io, &frame->carries[split - 1].f0.writer)) + split--; + for (size_t k = start; k < end; k++) + { + const TxnRewriteCarry *carry = &frame->carries[k]; + + if (!txn_rewrite_carried_push(frame, k < split ? &frame->theirs.carries : &frame->own, &carry, + sizeof(carry), _Alignof(const TxnRewriteCarry *))) + return; + } + start = end; + } + if (frame->theirs.spaces.len > 0 || frame->theirs.carries.len > 0) + { + for (size_t k = 0; k < frame->theirs.carries.len; k++) + { + const TxnRecordRowChange *change = &STD_VEC_AT(&frame->theirs.carries, const TxnRewriteCarry *, k)->f0; + const TxnMemtablePut *put; + TxnInsertCarriedPut carried; + + if (change->has_index || !txn_record_RowChange_put(change, &put)) + continue; + carried = (TxnInsertCarriedPut) + { + put->tuple, put->heap_bytes, change->writer + }; + if (!txn_rewrite_carried_push(frame, &frame->theirs.puts, &carried, sizeof(carried), + _Alignof(TxnInsertCarriedPut))) + return; + } + if (frame->page != NULL) + { + std_vec_free_Checked(&frame->theirs.takes, sizeof(TxnInsertCarriedTake), _Alignof(TxnInsertCarriedTake)); + if (!txn_insert_carried_takes(frame->io, frame->store, frame->held, frame->theirs.puts.ptr, + frame->theirs.puts.len, frame->page, &frame->theirs.takes, frame->error)) + { + std_vec_new(&frame->theirs.takes, sizeof(TxnInsertCarriedTake), _Alignof(TxnInsertCarriedTake)); + return; + } + } + if (!txn_rewrite_writers(frame) || + !txn_spill_carried(frame->io, frame->held, frame->writers.ptr, frame->writers.len, + txn_rewrite_theirs_closure_0, &frame->theirs, frame->error)) + return; + } + for (size_t s = 0; s < frame->own_spaces.len; s++) + if (!txn_rewrite_spaced(frame->io, frame->held, &STD_VEC_AT(&frame->own_spaces, TxnRecordMapCarried, s), + frame->logged, frame->most, false, 0, frame->error)) + return; + for (size_t k = 0; k < frame->own.len; k++) + { + const TxnRewriteCarry *carry = STD_VEC_AT(&frame->own, const TxnRewriteCarry *, k); + + if (!txn_rewrite_own_carried(frame->io, frame->store, frame->held, &carry->f0, carry->f1, carry->f2, + frame->logged, frame->page, frame->error)) + return; + } + frame->ok = true; +} + +/* + * txn_rewrite_carried carries a rewrite's, a move's, or an index build's + * versions into its new storage (P2), each key's chain, its map versions' + * and its rows', given oldest first: every version below a chain's newest + * run of the carrying transaction's own goes, with a rewrite's takes of the + * places of other writers' puts (insert's carried_takes), into spill files + * under each writer's top-level xid (spill's carried), disk to disk, so the + * carry never waits at the delta's budget on versions it can't spill; then + * each chain's own run goes into the memtable, a key's map versions as + * spaced pushes them and its rows as own_carried does, above them. Under the + * spill files' hold it logs, for a logged storage, each spilled key's map + * versions as one record and each spilled row version's own record, whose + * end each version keeps as its LSN; a lock is never logged. It leaves the + * new storage's totals unknown, since no commit counts a carried version + * (bugs row 316). + */ +bool +txn_rewrite_carried(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, + const TxnRecordMapCarried *spaces, size_t spaces_len, const TxnRewriteCarry *carries, + size_t carries_len, bool logged, const TxnInsertPage *page, uint16 most, HostError *error) +{ + txn_rewrite_carried_frame_1_t frame = {.io = io, .store = store, .held = held, .spaces = spaces, + .spaces_len = spaces_len, .carries = carries, .carries_len = carries_len, .logged = logged, .page = page, + .most = most, .error = error}; + + std_landing(&frame, txn_rewrite_carried_body_1, txn_rewrite_carried_pad_1); + txn_rewrite_carried_pad_1(&frame); + if (frame.ok) + txn_state_totals_unknown(host_shared_Held_record(held)); + return frame.ok; +} + +/* + * txn_rewrite_moved_point returns the fold point of a move's new base (P2): + * the old base's, or just below the commit end of the oldest writer of a + * carried version that committed at or below it, which only a map version a + * fold kept above an older running writer's has, so no carried spill file + * needs a residual's mark. The base holds the old one's content, and so + * commits past that point, but only those with no version left in the new + * storage's spills or memtable, whose files the old fold took out; nothing + * applies them twice. + */ +bool +txn_rewrite_moved_point(HostIoIo *io, HostLsn folded, const uint64 *writers, size_t writers_len, HostLsn *result, + HostError *error) +{ + *result = folded; + for (size_t k = 0; k < writers_len; k++) + { + HostXid xid = {writers[k]}; + TxnStatusCommit commit; + + if (!txn_status_committed(io, xid, &commit, error)) + return false; + if (commit.tag == TXN_STATUS_COMMIT_AT && commit.u.at <= folded) + *result = Min(*result, commit.u.at > 0 ? commit.u.at - 1 : 0); + } + return true; +} + /* txn_rewrite_value_map_frame_t is value_map's state across its pad: the trees it builds, unless it answers them. */ typedef struct txn_rewrite_value_map_frame_t { @@ -1264,7 +1807,11 @@ txn_rewrite_spaces(HostIoIo *io, const HostSharedHeld *held, HostStorage storage return frame.ok; } -/* txn_rewrite_carried_value is a carried map version's step: its entry put, or its removal for none. */ +/* + * txn_rewrite_carried_value is a carried map version's step: its entry put, + * or its removal for none. It reads no current entry, so its take has no + * read to compare (bugs row 342). + */ static TxnMemtableMapStep txn_rewrite_carried_value(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out) { @@ -1289,10 +1836,18 @@ txn_rewrite_nothing(void *captures) { } +/* txn_rewrite_Room_t is a carried version's writer's room, which it counts at, or none for a writer with no entry. */ +typedef struct txn_rewrite_Room_t +{ + bool has_roomed; + TxnStatusRoomed roomed; +} txn_rewrite_Room_t; + /* txn_rewrite_spaced_closure_2_captures_t is spaced's prepare's closure's captures. */ typedef struct txn_rewrite_spaced_closure_2_captures_t { const TxnRecordMapCarried *carried; + const StdVec *rooms; const StdVec *body; bool has_replayed; uint64 replayed; @@ -1338,19 +1893,21 @@ txn_rewrite_spaced_closure_2_body(void *frame_) for (size_t at = 0; at < c->carried->versions.len; at++) { const TxnRecordCarriedTake *version = &STD_VEC_AT(&c->carried->versions, TxnRecordCarriedTake, at); + const txn_rewrite_Room_t *room = &STD_VEC_AT(c->rooms, txn_rewrite_Room_t, at); if (version->has_mark) { StdVec mark; std_slice_to_vec_Checked(&mark, version->mark.ptr, version->mark.len, TXN_REWRITE_ID); - if (!txn_memtable_map_named(io, frame->prepared, at, version->writer, has_lsn, lsn, version->tuple, &mark, - frame->error)) + if (!txn_memtable_map_named(io, frame->prepared, at, version->writer, room->has_roomed, room->roomed, + has_lsn, lsn, version->tuple, &mark, frame->error)) return; continue; } - if (!txn_memtable_map_took_Unit(io, frame->prepared, at, version->writer, has_lsn, lsn, false, unclaimed, 0, - false, NULL, 0, txn_rewrite_carried_value, (void *) version, NULL, + if (!txn_memtable_map_took_Unit(io, frame->prepared, at, version->writer, room->has_roomed, room->roomed, + has_lsn, lsn, false, unclaimed, 0, false, NULL, 0, NULL, + txn_rewrite_carried_value, (void *) version, NULL, txn_rewrite_carried_tuple, (void *) version, NULL, txn_rewrite_nothing, NULL, NULL, &taken, frame->error)) return; @@ -1399,21 +1956,27 @@ typedef struct txn_rewrite_spaced_frame_t uint64 replayed; bool takes_live; StdVec takes; + bool rooms_live; + StdVec rooms; bool body_live; StdVec body; HostError *error; bool ok; } txn_rewrite_spaced_frame_t; -/* txn_rewrite_spaced_body readies a take for each carried version, runs the prepare, and stores each stored ender. */ +/* + * txn_rewrite_spaced_body readies a take for each carried version and its + * writer's room when its writer has an entry, runs the prepare, and stores + * each stored ender. + */ static void txn_rewrite_spaced_body(void *frame_) { txn_rewrite_spaced_frame_t *frame = frame_; const TxnRecordMapCarried *carried = frame->carried; HostSharedRecord record = host_shared_Held_record(frame->held); - txn_rewrite_spaced_closure_2_captures_t captures = {.carried = carried, .has_replayed = frame->has_replayed, - .replayed = frame->replayed}; + txn_rewrite_spaced_closure_2_captures_t captures = {.carried = carried, .rooms = &frame->rooms, + .has_replayed = frame->has_replayed, .replayed = frame->replayed}; size_t most; if (frame->has_replayed) @@ -1441,6 +2004,18 @@ txn_rewrite_spaced_body(void *frame_) std_vec_push_Checked(&frame->takes, &take, sizeof(take), _Alignof(TxnMemtableMapTake)); } + std_vec_with_capacity_Checked(&frame->rooms, carried->versions.len, sizeof(txn_rewrite_Room_t), + _Alignof(txn_rewrite_Room_t)); + frame->rooms_live = true; + for (size_t i = 0; i < carried->versions.len; i++) + { + const TxnRecordCarriedTake *version = &STD_VEC_AT(&carried->versions, TxnRecordCarriedTake, i); + txn_rewrite_Room_t room = {0}; + + if (!txn_status_kept_roomed(frame->io, &version->writer, &room.has_roomed, &room.roomed, frame->error)) + return; + std_vec_push_Checked(&frame->rooms, &room, sizeof(room), _Alignof(txn_rewrite_Room_t)); + } if (frame->logged && !frame->has_replayed) { txn_record_MapCarried_encode(carried, &frame->body); @@ -1455,7 +2030,8 @@ txn_rewrite_spaced_body(void *frame_) const TxnRecordCarriedTake *version = &STD_VEC_AT(&carried->versions, TxnRecordCarriedTake, i); if (version->ender != 0 && - !txn_memtable_map_ended(frame->io, record, version->tuple, (uint32) version->ender, frame->error)) + !txn_memtable_map_ended(frame->io, record, version->tuple, version->writer.xid, (uint32) version->ender, + frame->error)) return; } frame->ok = true; @@ -1471,7 +2047,9 @@ txn_rewrite_spaced_pad(void *frame_) std_vec_free_Checked(&frame->body, 1, 1); if (frame->takes_live) std_vec_free_Checked(&frame->takes, sizeof(TxnMemtableMapTake), _Alignof(TxnMemtableMapTake)); - frame->body_live = frame->takes_live = false; + if (frame->rooms_live) + std_vec_free_Checked(&frame->rooms, sizeof(txn_rewrite_Room_t), _Alignof(txn_rewrite_Room_t)); + frame->body_live = frame->takes_live = frame->rooms_live = false; } /* @@ -1479,9 +2057,9 @@ txn_rewrite_spaced_pad(void *frame_) * storage's new storage, in one checkpoint window outside redo: a logged * storage's record first, whose end each version keeps as its LSN, or redo's * record's end, then each version as its put of its entry, its removal, or - * its values mark, with its writer and tuple id, and then each take's or - * mark's stored ender. Redo passes over a key whose chain holds the record's - * LSN already. + * its values mark, with its writer and tuple id, counted at its writer's + * entry when the writer has one, and then each take's or mark's stored + * ender. Redo passes over a key whose chain holds the record's LSN already. */ bool txn_rewrite_spaced(HostIoIo *io, const HostSharedHeld *held, const TxnRecordMapCarried *carried, bool logged, diff --git a/src/doltgres/txn/c/roots.c b/src/doltgres/txn/c/roots.c index 1f716fd2e08ec..d0cf768f9aa3d 100644 --- a/src/doltgres/txn/c/roots.c +++ b/src/doltgres/txn/c/roots.c @@ -423,7 +423,7 @@ txn_roots_chunk_spills(txn_roots_frame_0_t *frame) if (!txn_spill_next(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, frame->io, &frame->hold, record, TXN_RUNS_KIND_SPILLED_CHUNKS, frame->after_live, frame->after.ptr, frame->after.len, - TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, &some, &frame->spilled, frame->error)) + TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, NULL, &some, &frame->spilled, frame->error)) return false; if (!some) return true; diff --git a/src/doltgres/txn/c/runs.c b/src/doltgres/txn/c/runs.c index 8eef1195a3b70..52f1b7f6b5701 100644 --- a/src/doltgres/txn/c/runs.c +++ b/src/doltgres/txn/c/runs.c @@ -116,7 +116,8 @@ txn_runs_Kind_of(uint8 number, TxnRunsKind *result) static const TxnRunsKind kinds[] = { TXN_RUNS_KIND_ROWS, TXN_RUNS_KIND_FREE, TXN_RUNS_KIND_VALUES, TXN_RUNS_KIND_CHUNKS, TXN_RUNS_KIND_STATUS, TXN_RUNS_KIND_SPILLED_ROWS, TXN_RUNS_KIND_SPILLED_VALUES, TXN_RUNS_KIND_SPILLED_TUPLES, TXN_RUNS_KIND_ENTRIES, - TXN_RUNS_KIND_SPILLED_ENTRIES, TXN_RUNS_KIND_IDENTITY, TXN_RUNS_KIND_SPILLED_CHUNKS, + TXN_RUNS_KIND_SPILLED_ENTRIES, TXN_RUNS_KIND_IDENTITY, TXN_RUNS_KIND_SPILLED_FREE, + TXN_RUNS_KIND_SPILLED_CHUNKS, TXN_RUNS_KIND_SPILLED_IDENTITY, }; for (size_t k = 0; k < lengthof(kinds); k++) @@ -134,7 +135,8 @@ txn_runs_Kind_spilled(TxnRunsKind self) { return self == TXN_RUNS_KIND_SPILLED_ROWS || self == TXN_RUNS_KIND_SPILLED_VALUES || self == TXN_RUNS_KIND_SPILLED_TUPLES || self == TXN_RUNS_KIND_SPILLED_ENTRIES || - self == TXN_RUNS_KIND_SPILLED_CHUNKS; + self == TXN_RUNS_KIND_SPILLED_FREE || self == TXN_RUNS_KIND_SPILLED_CHUNKS || + self == TXN_RUNS_KIND_SPILLED_IDENTITY; } /* txn_runs_Names_PartialEq_eq is Names' derived PartialEq. */ @@ -637,6 +639,7 @@ txn_runs_refused(const TxnRunsNames *names, const TxnMemtableVersionRecord *vers case TXN_RUNS_KIND_ROWS: return emptied || named || ended || heaped_false; case TXN_RUNS_KIND_FREE: + case TXN_RUNS_KIND_SPILLED_FREE: { TxnMemtableVersionWriter writer = txn_memtable_version_Record_writer(version); TxnMemtableVersionWriter none = TXN_MEMTABLE_VERSION_NO_TRANSACTION; @@ -644,7 +647,8 @@ txn_runs_refused(const TxnRunsNames *names, const TxnMemtableVersionRecord *vers return !command || heaped_true || (ended && (ender < (uint32) TXN_STATUS_FIRST_NORMAL || txn_memtable_version_Writer_PartialEq_eq(&writer, &none) || - txn_memtable_map_taken(op).f0 == 0)); + txn_memtable_map_taken(op).f0 == 0)) || + (names->kind == TXN_RUNS_KIND_SPILLED_FREE && txn_runs_top(version) != names->redo); } case TXN_RUNS_KIND_VALUES: return emptied || named || ended || !command || heaped_true; @@ -676,6 +680,12 @@ txn_runs_refused(const TxnRunsNames *names, const TxnMemtableVersionRecord *vers if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_PUT) return op.u.put.heap_bytes != 0 || op.u.put.tuple.f0 != 0; return op.tag != TXN_MEMTABLE_VERSION_OP_REF_DELETE; + case TXN_RUNS_KIND_SPILLED_IDENTITY: + if (txn_runs_top(version) != names->redo) + return true; + if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_PUT) + return op.u.put.heap_bytes != 0 || op.u.put.tuple.f0 != 0; + return op.tag != TXN_MEMTABLE_VERSION_OP_REF_DELETE; } return true; } @@ -2039,6 +2049,7 @@ txn_runs_tree_name(TxnRunsKind kind, bool *whole) case TXN_RUNS_KIND_SPILLED_ROWS: return "rows"; case TXN_RUNS_KIND_FREE: + case TXN_RUNS_KIND_SPILLED_FREE: return "free"; case TXN_RUNS_KIND_VALUES: case TXN_RUNS_KIND_SPILLED_VALUES: @@ -2053,6 +2064,7 @@ txn_runs_tree_name(TxnRunsKind kind, bool *whole) *whole = true; return "chunks"; case TXN_RUNS_KIND_IDENTITY: + case TXN_RUNS_KIND_SPILLED_IDENTITY: *whole = true; return "identity"; case TXN_RUNS_KIND_STATUS: @@ -2249,11 +2261,11 @@ txn_runs_Writing_path(const TxnRunsWriting *self) /* txn_runs_Writing_write writes the record the writing holds next at its place in its file, mapping a file's error to the storage's. */ static bool -txn_runs_Writing_write(TxnRunsWriting *self, HostIoIo *io, uint64 at, HostError *error) +txn_runs_Writing_write(TxnRunsWriting *self, HostIoIoDoors io, uint64 at, HostError *error) { ChunksErrorError failed; - if (!chunks_file_write_at(HOST_IO_IO_IO_DOORS(io), &self->file, self->path.bytes.ptr, self->path.bytes.len, at, + if (!chunks_file_write_at(io, &self->file, self->path.bytes.ptr, self->path.bytes.len, at, self->out.ptr, self->out.len, &failed)) { *error = txn_runs_storage(&failed); @@ -2264,21 +2276,21 @@ txn_runs_Writing_write(TxnRunsWriting *self, HostIoIo *io, uint64 at, HostError /* txn_runs_Writing_key appends a key's record to the file, with no lock held, making the file with its header at the first. */ bool -txn_runs_Writing_key(TxnRunsWriting *self, HostIoIo *io, const uint8 *key, size_t key_len, const StdBytes *records, +txn_runs_Writing_key(TxnRunsWriting *self, HostIoIoDoors io, const uint8 *key, size_t key_len, const StdBytes *records, size_t records_len, HostError *error) { ChunksErrorError failed; if (!self->has_file) { - if (!chunks_file_create(HOST_IO_IO_IO_DOORS(io), self->path.bytes.ptr, self->path.bytes.len, &self->file, + if (!chunks_file_create(io, self->path.bytes.ptr, self->path.bytes.len, &self->file, &failed)) { *error = txn_runs_storage(&failed); return false; } self->has_file = true; - if (!chunks_file_write_at(HOST_IO_IO_IO_DOORS(io), &self->file, self->path.bytes.ptr, self->path.bytes.len, 0, + if (!chunks_file_write_at(io, &self->file, self->path.bytes.ptr, self->path.bytes.len, 0, self->head.ptr, self->head.len, &failed)) { self->has_file = false; @@ -2303,7 +2315,7 @@ txn_runs_Writing_key(TxnRunsWriting *self, HostIoIo *io, const uint8 *key, size_ * file. Its owner drops the writing after. */ bool -txn_runs_Writing_end(TxnRunsWriting *self, HostIoIo *io, const uint64 *filter, size_t filter_len, bool *some, +txn_runs_Writing_end(TxnRunsWriting *self, HostIoIoDoors io, const uint64 *filter, size_t filter_len, bool *some, RepoRunFile *result, HostError *error) { ChunksErrorError failed; @@ -2316,7 +2328,7 @@ txn_runs_Writing_end(TxnRunsWriting *self, HostIoIo *io, const uint64 *filter, s hash = txn_runs_Encoder_end(&self->encoder, filter, filter_len, &self->out); if (!txn_runs_Writing_write(self, io, self->at, error)) return false; - if (!chunks_file_sync(HOST_IO_IO_IO_DOORS(io), &self->file, &failed)) + if (!chunks_file_sync(io, &self->file, &failed)) { *error = txn_runs_storage(&failed); return false; @@ -2327,8 +2339,10 @@ txn_runs_Writing_end(TxnRunsWriting *self, HostIoIo *io, const uint64 *filter, s result->len = STD_ADD(u64, self->at, (uint64) self->out.len, 961, 41); result->hash = hash; result->has_spilled = self->has_number; - result->spilled.xid = self->names.redo; - result->spilled.number = self->number; + result->spilled = (RepoSpilled) + { + .xid = self->names.redo, .number = self->number + }; self->has_file = false; host_File_drop(&self->file); *some = true; @@ -2435,13 +2449,13 @@ txn_runs_statuses_body_0(void *frame_) std_u64_to_le_bytes(word, value); std_u64_to_le_bytes(count, value + 8); std_u64_to_be_bytes(xid, key); - if (!txn_runs_Writing_key(frame->writing, frame->io, key, sizeof(key), &record, 1, frame->error)) + if (!txn_runs_Writing_key(frame->writing, HOST_IO_IO_IO_DOORS(frame->io), key, sizeof(key), &record, 1, frame->error)) { txn_runs_statuses_pad_0(frame); return; } } - frame->ok = txn_runs_Writing_end(frame->writing, frame->io, NULL, 0, frame->some, frame->result, frame->error); + frame->ok = txn_runs_Writing_end(frame->writing, HOST_IO_IO_IO_DOORS(frame->io), NULL, 0, frame->some, frame->result, frame->error); txn_runs_statuses_pad_0(frame); } @@ -2530,7 +2544,7 @@ txn_runs_written_key(txn_runs_written_frame_0_t *frame, HostIoIo *io, const uint HostError *error) { return txn_runs_tops(frame->records.ptr, frame->records.len, frame->xids, error) && - txn_runs_Writing_key(frame->writing, io, key, key_len, frame->records.ptr, frame->records.len, error); + txn_runs_Writing_key(frame->writing, HOST_IO_IO_IO_DOORS(io), key, key_len, frame->records.ptr, frame->records.len, error); } /* txn_runs_written_gather gathers a key's records into the frame's records, borrowing them. */ @@ -2561,7 +2575,7 @@ txn_runs_written_each(void *captures, HostIoIo *io, const uint8 *key, size_t key static void txn_runs_written_end(txn_runs_written_frame_0_t *frame) { - frame->ok = txn_runs_Writing_end(frame->writing, frame->io, NULL, 0, frame->some, frame->result, frame->error); + frame->ok = txn_runs_Writing_end(frame->writing, HOST_IO_IO_IO_DOORS(frame->io), NULL, 0, frame->some, frame->result, frame->error); txn_runs_written_pad_0(frame); } @@ -2828,18 +2842,32 @@ txn_runs_write_file(txn_runs_write_frame_0_t *frame, bool some, const RepoRunFil std_vec_push_Checked(&frame->files, file, sizeof(*file), _Alignof(RepoRunFile)); } -/* txn_runs_write_named adds the spill files a home's manifest and its processes' slots name, a checkpoint's, to the frame's files, and their xids. */ +/* + * txn_runs_write_named adds the spill files a home's manifest and its + * processes' slots name, a checkpoint's, to the frame's files, and their + * xids; an identity spill the checkpoint's save folds at its point is left + * out, as the identity run leaves out such versions (bugs row 291). + */ static bool txn_runs_write_named(txn_runs_write_frame_0_t *frame, const TxnSpillHomeOps *home, HostSharedRecord record, const HostStorage *storage) { - if (!txn_spill_named(home, frame->io, record, storage, &frame->named, frame->error)) + if (!txn_spill_named(home, frame->io, record, storage, frame->redo, &frame->named, frame->error)) return false; frame->named_live = true; for (size_t k = 0; k < frame->named.len; k++) { const RepoRunFile *file = &STD_VEC_AT(&frame->named, RepoRunFile, k); + if (home == &TXN_SPILL_IDENTITY_HOME_OPS && file->has_spilled) + { + bool applied; + + if (!txn_spill_identity_applied(frame->io, file->spilled.xid, frame->point, &applied, frame->error)) + return false; + if (applied) + continue; + } if (file->has_spilled) { if (!std_vec_try_reserve(&frame->xids, 1, sizeof(uint64), _Alignof(uint64))) @@ -2924,6 +2952,7 @@ txn_runs_write_deltas(txn_runs_write_frame_0_t *frame) .database = frame->repository, .redo = frame->redo}; TxnRunsTail none = {.tag = TXN_RUNS_TAIL_NONE}; HostStorage chunks = txn_spill_chunks_storage(frame->repository); + HostStorage identity = txn_spill_identity_storage(frame->repository); bool some; RepoRunFile file; @@ -2932,7 +2961,8 @@ txn_runs_write_deltas(txn_runs_write_frame_0_t *frame) return false; txn_runs_write_file(frame, some, &file); if (!txn_spill_chunks_ended(frame->io, record, frame->error) || - !txn_runs_write_named(frame, &TXN_SPILL_BRANCH_DELTAS_HOME_OPS, record, &chunks)) + !txn_runs_write_named(frame, &TXN_SPILL_BRANCH_DELTAS_HOME_OPS, record, &chunks) || + !txn_runs_write_named(frame, &TXN_SPILL_IDENTITY_HOME_OPS, record, &identity)) return false; names.kind = TXN_RUNS_KIND_IDENTITY; if (!txn_runs_unfolded(frame->io, txn_runs_write_dir(frame), names, record, frame->point, &frame->xids, &some, @@ -3085,7 +3115,7 @@ txn_runs_host(HostError error) return wrapped; } -/* txn_runs_repo_frame_0_t is the state of cleaned and checkpoint across their pad: a repository open, its reader, and what each holds of it, while live. */ +/* txn_runs_repo_frame_0_t is the state of cleaned, checkpoint, and save across their pad: a repository open, its reader, and what each holds of it, while live. */ typedef struct txn_runs_repo_frame_0_t { HostIoIo *io; @@ -3126,6 +3156,16 @@ typedef struct txn_runs_repo_frame_0_t bool pairs_live; size_t moved_pairs; uint32 database; + + /* + * own is save's repository's own tablespace, took whether its drop took + * an identity version out, and cleaned whether its marks turned a state + * it gathered clean, which save answers through cleaned_out. + */ + uint32 own; + bool *took; + bool cleaned; + bool *cleaned_out; HostLsn point; } txn_runs_repo_frame_0_t; @@ -3511,12 +3551,19 @@ txn_runs_cleaned(HostIoIo *io, HostError *error) return frame.ok; } -/* txn_runs_gathering is checkpoint's gathering for its repository's save: the states' edits, kept for the marks, and the identity tree's folded pairs. */ +/* + * txn_runs_gathering is checkpoint's gathering for its repository's save: the + * states' edits, kept for the marks, and the identity tree's folded pairs, + * with the WAL flushed through the latest record end of the bases gathered, + * and through the point when it folded a pair, so the manifest names nothing + * a crash loses (bugs row 350). + */ static bool txn_runs_gathering(void *captures, HostIoIo *io, const RepoRead *read, StdVec *result, ChunksErrorError *error) { txn_runs_repo_frame_0_t *frame = captures; HostError failed; + HostLsn flushed; if (frame->saving_live) { @@ -3553,8 +3600,14 @@ txn_runs_gathering(void *captures, HostIoIo *io, const RepoRead *read, StdVec *r frame->moved_pairs = std_wrapping_add_usize(frame->moved_pairs, 1); std_vec_push_Checked(&frame->edits, &edit, sizeof(edit), _Alignof(RepoGathered)); } + flushed = Max(frame->saving.end, frame->pairs.len > 0 ? frame->point : 0); frame->pairs_live = false; std_vec_free_Checked(&frame->pairs, sizeof(TxnIdentityPair), _Alignof(TxnIdentityPair)); + if (flushed > 0 && !host_wal_flush(HOST_IO_IO_IO_DOORS(io), flushed, &failed)) + { + *error = txn_runs_host(failed); + return false; + } *result = frame->edits; frame->edits_live = false; return true; @@ -3571,6 +3624,7 @@ txn_runs_marks(void *captures, HostIoIo *io, ChunksErrorError *error) if (!frame->saving_live) return true; frame->saving_live = false; + frame->cleaned = frame->saving.locations.len > 0; saved = txn_state_Saving_saved(&frame->saving, io, &failed); if (!saved) *error = txn_runs_host(failed); @@ -3586,6 +3640,7 @@ txn_runs_checkpoint_repository(txn_runs_repo_frame_0_t *frame, uint32 database, { HostLsn position; ChunksErrorError failed; + bool took; frame->database = database; if (database == 0) @@ -3625,11 +3680,59 @@ txn_runs_checkpoint_repository(txn_runs_repo_frame_0_t *frame, uint32 database, *frame->error = txn_base_storage(&failed); return false; } - return txn_identity_dropped(frame->io, database, frame->point, frame->error) && + return txn_identity_dropped(frame->io, database, frame->point, &took, frame->error) && txn_runs_removed(frame->io, &frame->repo, database, own, TXN_RUNS_REMOVAL_CHECKPOINT, frame->error) && txn_runs_repo_close(frame); } +/* txn_runs_save_body_0 is save's body: the repository's save at a fold point of its own, its runs field as read. */ +static void +txn_runs_save_body_0(void *frame_) +{ + txn_runs_repo_frame_0_t *frame = frame_; + HostLsn position; + ChunksErrorError failed; + + if (!txn_runs_repo_open(frame, frame->database, frame->own) || + !host_fold_point(HOST_IO_IO_IO_DOORS(frame->io), &position, frame->error) || + !txn_status_lowered(frame->io, position, &frame->point, frame->error) || + !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-runs-save-pointed", frame->error)) + { + txn_runs_repo_pad_0(frame); + return; + } + if (!repo_Repo_save(&frame->repo, frame->io, &frame->reading, false, NULL, txn_runs_gathering, frame, + txn_runs_marks, frame, &failed)) + { + *frame->error = txn_base_storage(&failed); + txn_runs_repo_pad_0(frame); + return; + } + frame->ok = txn_identity_dropped(frame->io, frame->database, frame->point, frame->took, frame->error) && + txn_runs_repo_close(frame); + *frame->cleaned_out = frame->cleaned; + txn_runs_repo_pad_0(frame); +} + +/* + * txn_runs_save is item 6's save outside a checkpoint, which a states' + * pool's make with nothing to free and a writer the budget refused run + * (bugs row 340): it saves a repository's unsaved bases and its identity + * tree's committed versions at a fold point of its own, writes the runs field + * exactly as read, and drops what the save folded of the identity tree, + * answering whether that took a version or an identity spill out, and + * whether its marks turned a state it gathered clean. + */ +bool +txn_runs_save(HostIoIo *io, uint32 repository, uint32 own, bool *took, bool *cleaned, HostError *error) +{ + txn_runs_repo_frame_0_t frame = {.io = io, .database = repository, .own = own, .took = took, + .cleaned_out = cleaned, .error = error}; + + std_landing(&frame, txn_runs_save_body_0, txn_runs_repo_pad_0); + return frame.ok; +} + static bool txn_runs_saved(HostIoIo *io, StdVec *result, HostError *error); /* txn_runs_checkpoint_body_0 is checkpoint's body. */ @@ -3792,72 +3895,78 @@ txn_runs_saved(HostIoIo *io, StdVec *result, HostError *error) return frame.ok; } -/* txn_TuplePathBufU64_t is DEFERRED's (PathBuf, u64): a spill file and the epoch a checkpoint found it at. */ -typedef struct txn_TuplePathBufU64_t +/* txn_TuplePathBufU64Bool_t is DEFERRED's (PathBuf, u64, bool): a spill file, the epoch a checkpoint found it at, and whether a checkpoint found it again. */ +typedef struct txn_TuplePathBufU64Bool_t { StdPathBuf f0; uint64 f1; -} txn_TuplePathBufU64_t; + bool f2; +} txn_TuplePathBufU64Bool_t; -/* txn_runs_TuplePathBufU64_drop is (PathBuf, u64)'s drop glue: its path's. */ +/* txn_runs_TuplePathBufU64Bool_drop is (PathBuf, u64, bool)'s drop glue: its path's. */ static void -txn_runs_TuplePathBufU64_drop(void *elem) +txn_runs_TuplePathBufU64Bool_drop(void *elem) { - txn_TuplePathBufU64_t *self = elem; + txn_TuplePathBufU64Bool_t *self = elem; std_vec_free_Checked(&self->f0.bytes, 1, 1); } -/* txn_runs_VecTuplePathBufU64_drop is Vec<(PathBuf, u64)>'s drop glue. */ +/* txn_runs_VecTuplePathBufU64Bool_drop is Vec<(PathBuf, u64, bool)>'s drop glue. */ static void -txn_runs_VecTuplePathBufU64_drop(void *value) +txn_runs_VecTuplePathBufU64Bool_drop(void *value) { - std_vec_drop_Checked(value, sizeof(txn_TuplePathBufU64_t), _Alignof(txn_TuplePathBufU64_t), - txn_runs_TuplePathBufU64_drop); + std_vec_drop_Checked(value, sizeof(txn_TuplePathBufU64Bool_t), _Alignof(txn_TuplePathBufU64Bool_t), + txn_runs_TuplePathBufU64Bool_drop); } -/* taken at Vec<(PathBuf, u64)> and IoMutex>, which runs.c alone uses. */ +/* taken at Vec<(PathBuf, u64, bool)> and IoMutex>, which runs.c alone uses. */ #define HOST_LOCK_SCOPE static #define HOST_LOCK_DECLARE #define HOST_LOCK_DEFINE -#define HOST_LOCK_TAKEN_TS VecTuplePathBufU64 +#define HOST_LOCK_TAKEN_TS VecTuplePathBufU64Bool #define HOST_LOCK_TAKEN_T StdVec #define HOST_LOCK_TAKEN_DEFAULT(place) \ - std_vec_new((place), sizeof(txn_TuplePathBufU64_t), _Alignof(txn_TuplePathBufU64_t)) -#define HOST_LOCK_TAKEN_DROP(place) txn_runs_VecTuplePathBufU64_drop(place) + std_vec_new((place), sizeof(txn_TuplePathBufU64Bool_t), _Alignof(txn_TuplePathBufU64Bool_t)) +#define HOST_LOCK_TAKEN_DROP(place) txn_runs_VecTuplePathBufU64Bool_drop(place) #define HOST_LOCK_TAKEN_TYPE TXN_RUNS_DEFERRED_LOCK_TYPE #define HOST_LOCK_IO_MUTEX -#define HOST_LOCK_TS VecTuplePathBufU64 +#define HOST_LOCK_TS VecTuplePathBufU64Bool #define HOST_LOCK_T StdVec #define HOST_LOCK_DEFAULT(place) \ - std_vec_new((place), sizeof(txn_TuplePathBufU64_t), _Alignof(txn_TuplePathBufU64_t)) + std_vec_new((place), sizeof(txn_TuplePathBufU64Bool_t), _Alignof(txn_TuplePathBufU64Bool_t)) #include "doltgres/host/lock_template.h" -/* txn_runs_IoMutexVecTuplePathBufU64_drop is IoMutex>' drop glue: its value's. */ +/* txn_runs_IoMutexVecTuplePathBufU64Bool_drop is IoMutex>' drop glue: its value's. */ static void -txn_runs_IoMutexVecTuplePathBufU64_drop(void *value) +txn_runs_IoMutexVecTuplePathBufU64Bool_drop(void *value) { - HostLockIoMutexVecTuplePathBufU64 *self = value; + HostLockIoMutexVecTuplePathBufU64Bool *self = value; - txn_runs_VecTuplePathBufU64_drop(&self->value); + txn_runs_VecTuplePathBufU64Bool_drop(&self->value); } -/* TXN_RUNS_DEFERRED_TYPE is IoMutex>' size, Default, and drop. */ -static const StdDefaultType TXN_RUNS_DEFERRED_TYPE = {sizeof(HostLockIoMutexVecTuplePathBufU64), - _Alignof(HostLockIoMutexVecTuplePathBufU64), host_lock_IoMutexVecTuplePathBufU64_Default_default, -txn_runs_IoMutexVecTuplePathBufU64_drop}; +/* TXN_RUNS_DEFERRED_TYPE is IoMutex>' size, Default, and drop. */ +static const StdDefaultType TXN_RUNS_DEFERRED_TYPE = {sizeof(HostLockIoMutexVecTuplePathBufU64Bool), + _Alignof(HostLockIoMutexVecTuplePathBufU64Bool), host_lock_IoMutexVecTuplePathBufU64Bool_Default_default, +txn_runs_IoMutexVecTuplePathBufU64Bool_drop}; /* * TXN_RUNS_DEFERRED is DEFERRED: the checkpointer's spill files that no * manifest names and whose transactions ended, each with the epoch a * checkpoint found it at, which a later checkpoint removes once every reader * that may have listed it ended; its BackendLocal header and the process's - * value, which follows it. + * value, which follows it. A fold's residual listed after a checkpoint named + * its storage's files is unnamed at that checkpoint while its manifest lists + * it (bugs row 228), so a file's epoch is the one of the second checkpoint in + * a row that found it unnamed, whose naming came after its unlisting, and + * whether a checkpoint found it again; a file a later checkpoint names + * leaves the list. */ static struct { HostLocalBackendLocal local; - HostLockIoMutexVecTuplePathBufU64 process; + HostLockIoMutexVecTuplePathBufU64Bool process; } TXN_RUNS_DEFERRED = {HOST_LOCAL_BACKEND_LOCAL_NEW}; /* txn_runs_removed_frame_0_t is removed's state across its pad: the manifests, the directories and their names, the kept spills, and the deferred files' guard, while live. */ @@ -4033,7 +4142,7 @@ txn_runs_removed_kept(txn_runs_removed_frame_0_t *frame, const RepoRuns *const * return true; } -/* txn_runs_removed_xid returns the xid a spill file's name holds, chunks' third word of three or a tree's fourth of five. */ +/* txn_runs_removed_xid returns the xid a spill file's name holds, chunks' or identity's third word of three or a tree's fourth of five. */ static bool txn_runs_removed_xid(const StdString *name, uint64 *xid) { @@ -4051,16 +4160,22 @@ txn_runs_removed_xid(const StdString *name, uint64 *xid) starts[words++] = std_wrapping_add_usize(k, 1); } starts[words] = std_wrapping_add_usize(name->len, 1); - if (words == 3 && starts[1] - 1 == strlen("chunks") && memcmp(bytes, "chunks", strlen("chunks")) == 0) + if (words == 3 && ((starts[1] - 1 == strlen("chunks") && memcmp(bytes, "chunks", strlen("chunks")) == 0) || + (starts[1] - 1 == strlen("identity") && memcmp(bytes, "identity", strlen("identity")) == 0))) return std_str_parse_u64(bytes + starts[1], starts[2] - starts[1] - 1, xid, &failed); if (words == 5) return std_str_parse_u64(bytes + starts[3], starts[4] - starts[3] - 1, xid, &failed); return false; } -/* txn_runs_removed_deferred reports whether a checkpoint removes a spill file of an ended transaction: one it deferred whose epoch every reader passed, deferring one it didn't. */ +/* + * txn_runs_removed_deferred reports whether a checkpoint removes a spill file + * of an ended transaction: one it found again whose epoch every reader + * passed, the epoch of the first finding again taking the first's, deferring + * one it didn't; a file a checkpoint names (kept) leaves the list. + */ static bool -txn_runs_removed_deferred(txn_runs_removed_frame_0_t *frame, uint64 passed, uint64 oldest, bool *removable) +txn_runs_removed_deferred(txn_runs_removed_frame_0_t *frame, bool kept, uint64 passed, uint64 oldest, bool *removable) { StdVec *deferred; size_t at; @@ -4074,7 +4189,7 @@ txn_runs_removed_deferred(txn_runs_removed_frame_0_t *frame, uint64 passed, uint deferred = host_lock_LockedRwLockWriteGuard_DerefMut_deref_mut(&frame->guard); for (at = 0; at < deferred->len; at++) { - const StdPathBuf *listed = &STD_VEC_AT(deferred, txn_TuplePathBufU64_t, at).f0; + const StdPathBuf *listed = &STD_VEC_AT(deferred, txn_TuplePathBufU64Bool_t, at).f0; if (std_path_PartialEq_eq(listed->bytes.ptr, listed->bytes.len, frame->path.bytes.ptr, frame->path.bytes.len)) break; @@ -4082,26 +4197,33 @@ txn_runs_removed_deferred(txn_runs_removed_frame_0_t *frame, uint64 passed, uint *removable = false; if (at < deferred->len) { - if (host_delta_page_passed(STD_VEC_AT(deferred, txn_TuplePathBufU64_t, at).f1, oldest)) + txn_TuplePathBufU64Bool_t *found = &STD_VEC_AT(deferred, txn_TuplePathBufU64Bool_t, at); + + if (kept || (found->f2 && host_delta_page_passed(found->f1, oldest))) { - txn_TuplePathBufU64_t removed; + txn_TuplePathBufU64Bool_t removed; - std_vec_swap_remove(deferred, at, &removed, sizeof(removed), _Alignof(txn_TuplePathBufU64_t)); + std_vec_swap_remove(deferred, at, &removed, sizeof(removed), _Alignof(txn_TuplePathBufU64Bool_t)); std_vec_free_Checked(&removed.f0.bytes, 1, 1); - *removable = true; + *removable = !kept; + } + else if (!found->f2) + { + found->f1 = passed; + found->f2 = true; } } - else + else if (!kept) { - txn_TuplePathBufU64_t listed = {.f1 = passed}; + txn_TuplePathBufU64Bool_t listed = {.f1 = passed}; - if (!std_vec_try_reserve(deferred, 1, sizeof(listed), _Alignof(txn_TuplePathBufU64_t))) + if (!std_vec_try_reserve(deferred, 1, sizeof(listed), _Alignof(txn_TuplePathBufU64Bool_t))) { *frame->error = host_Error_out_of_memory(); return false; } std_PathBuf_Clone_clone(&frame->path, &listed.f0); - std_vec_push_Checked(deferred, &listed, sizeof(listed), _Alignof(txn_TuplePathBufU64_t)); + std_vec_push_Checked(deferred, &listed, sizeof(listed), _Alignof(txn_TuplePathBufU64Bool_t)); } frame->guard_live = false; host_lock_LockedRwLockWriteGuard_drop(&frame->guard); @@ -4142,7 +4264,18 @@ txn_runs_removed_spills(txn_runs_removed_frame_0_t *frame) kept = txn_runs_string_eq(name, (const char *) spilled->ptr, spilled->len); } - if (kept || listing) + if (kept) + { + std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) name->ptr, name->len, + &frame->path); + frame->path_live = true; + if (!txn_runs_removed_deferred(frame, true, 0, 0, &removable)) + return false; + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + continue; + } + if (listing) continue; if (frame->at == TXN_RUNS_REMOVAL_CHECKPOINT && txn_runs_removed_xid(name, &xid)) { @@ -4164,7 +4297,7 @@ txn_runs_removed_spills(txn_runs_removed_frame_0_t *frame) host_delta_advanced(&passed, &oldest); advanced = true; } - if (!txn_runs_removed_deferred(frame, passed, oldest, &removable)) + if (!txn_runs_removed_deferred(frame, false, passed, oldest, &removable)) return false; } } @@ -4293,21 +4426,6 @@ txn_runs_tid(const TxnMemtableVersionOpRef *op, HostRow *result) } } -/* txn_runs_map_counted counts a map's or a database-level tree's version read back from a run at its writer's entry. */ -static bool -txn_runs_map_counted(HostIoIo *io, const TxnMemtableVersionRecord *version, HostError *error) -{ - TxnMemtableVersionWriter writer = txn_memtable_version_Record_writer(version); - TxnMemtableVersionWriter none = TXN_MEMTABLE_VERSION_NO_TRANSACTION; - HostXid top = {writer.has_top ? writer.top : writer.xid}; - bool has; - uint64 added; - - if (txn_memtable_version_Writer_PartialEq_eq(&writer, &none)) - return true; - return txn_status_add(io, top, 1, &has, &added, error); -} - /* txn_runs_pushed_back_frame_0_t is pushed_back's state across its pad: the version's op, owned, while live. */ typedef struct txn_runs_pushed_back_frame_0_t { @@ -4345,19 +4463,24 @@ txn_runs_pushed_back_body_0(void *frame_) .op = &frame->op, .lock = txn_memtable_version_Record_lock(frame->version), .shown = txn_memtable_version_Record_shown(frame->version)}; TxnMemtableVersionHandle handle; + bool has_roomed = false; + TxnStatusRoomed roomed = {0}; frame->op = txn_memtable_version_OpRef_owned(&op); frame->op_live = true; change.has_lsn = txn_memtable_version_Record_lsn(frame->version, &change.lsn); - frame->ok = txn_memtable_table_alone(frame->tree, frame->io, frame->owner, frame->key.ptr, frame->key.len, &change, - &handle, frame->error); + frame->ok = (!frame->tree->counted || + txn_status_kept_roomed(frame->io, &change.writer, &has_roomed, &roomed, frame->error)) && + txn_memtable_table_alone(frame->tree, frame->io, frame->owner, frame->key.ptr, frame->key.len, &change, + has_roomed, roomed, &handle, frame->error); txn_runs_pushed_back_pad_0(frame); } /* * txn_runs_pushed_back pushes a version read back from a run into one of an * owner's trees with no window, its writer, op, lock, and LSN as the run - * holds them. + * holds them, counted at its writer's entry when the tree counts and the + * writer has one. */ static bool txn_runs_pushed_back(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, StdBytes key, @@ -4462,14 +4585,20 @@ txn_runs_TupleBoolU32U32BoolBoolU8_lt(const txn_TupleBoolU32U32BoolBoolU8_t *a, #define STD_STABLE_SORT_KEY_LT(a, b) txn_runs_TupleBoolU32U32BoolBoolU8_lt((a), (b)) #include "doltgres/std/stable_sort_template.h" -/* txn_runs_pushed_runs_closure_0 is pushed_runs' sort key of a run file. */ +/* + * txn_runs_pushed_runs_closure_0 is pushed_runs' sort key of a run file: an + * identity run sorts by its tree's storage, so its spills, which that storage + * names, are linked before its versions push and spill (bugs row 291). + */ static txn_TupleBoolU32U32BoolBoolU8_t txn_runs_pushed_runs_closure_0(void *captures, const void *elem) { const RepoRunFile *file = elem; TxnRunsKind kind; bool spilled = txn_runs_Kind_of(file->kind, &kind) && txn_runs_Kind_spilled(kind); - txn_TupleBoolU32U32BoolBoolU8_t key = {file->kind != TXN_RUNS_KIND_STATUS, file->tablespace, file->relation, + uint32 relation = file->kind == TXN_RUNS_KIND_IDENTITY ? txn_spill_identity_storage(0).relation : + file->relation; + txn_TupleBoolU32U32BoolBoolU8_t key = {file->kind != TXN_RUNS_KIND_STATUS, file->tablespace, relation, file->kind != TXN_RUNS_KIND_FREE, !spilled, file->kind}; return key; @@ -4645,6 +4774,25 @@ txn_runs_pushed_status(txn_runs_pushed_runs_frame_0_t *frame, const TxnRunsNames return true; } +/* + * txn_runs_pushed_identity counts an identity run's version a start pushes + * toward its transaction's replayed bytes, which spill past the threshold as + * redo's do, so a run larger than the budget loads (bugs row 291). + */ +static bool +txn_runs_pushed_identity(txn_runs_pushed_runs_frame_0_t *frame, const TxnMemtableVersionRecord *version) +{ + HostStorage storage = txn_spill_identity_storage(frame->repository); + uint64 top = txn_runs_top(version); + uint64 counted; + bool spilled; + + return txn_spill_replayed(frame->io, &storage, TXN_WRITTEN_OF_IDENTITY, top, + txn_memtable_version_Record_bytes(version).len, &counted, frame->error) && + (counted < txn_spill_threshold() || + txn_spill_spilled(frame->io, &storage, TXN_WRITTEN_OF_IDENTITY, top, &spilled, frame->error)); +} + /* txn_runs_pushed_branch pushes a working state's chunks or identity run, or links its chunk spill. */ static bool txn_runs_pushed_branch(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind, const RepoRunFile *file) @@ -4655,16 +4803,18 @@ txn_runs_pushed_branch(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind, return false; frame->deltas_live = true; record = host_shared_Held_record(&frame->deltas); - if (kind == TXN_RUNS_KIND_SPILLED_CHUNKS) + if (kind == TXN_RUNS_KIND_SPILLED_CHUNKS || kind == TXN_RUNS_KIND_SPILLED_IDENTITY) { TxnSpillFile listing = {.xid = file->spilled.xid, .kind = kind, .number = file->spilled.number, .len = file->len, .hash = file->hash}; - HostStorage storage = txn_spill_chunks_storage(frame->repository); + bool chunks = kind == TXN_RUNS_KIND_SPILLED_CHUNKS; + HostStorage storage = chunks ? txn_spill_chunks_storage(frame->repository) : + txn_spill_identity_storage(frame->repository); txn_TupleU64KindU32OptionU32_t none = {0}; return txn_runs_spill_bits(frame->bytes.ptr, frame->bytes.len, &listing.bits, frame->error) && - txn_spill_linked(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, frame->io, record, &storage, &listing, 1, false, none, - frame->error); + txn_spill_linked(chunks ? &TXN_SPILL_BRANCH_DELTAS_HOME_OPS : &TXN_SPILL_IDENTITY_HOME_OPS, frame->io, + record, &storage, &listing, 1, false, none, frame->error); } for (size_t k = 0; k < frame->decoded.keys.len; k++) { @@ -4674,10 +4824,11 @@ txn_runs_pushed_branch(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind, { const TxnMemtableVersionRecord *version = &STD_VEC_AT(&key->versions, TxnMemtableVersionRecord, v - 1); + if (kind == TXN_RUNS_KIND_IDENTITY && !txn_runs_pushed_identity(frame, version)) + return false; if (!txn_runs_pushed_back(kind == TXN_RUNS_KIND_CHUNKS ? &TXN_MEMTABLE_BRANCH_CHUNKS_TREE_OPS : &TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS, frame->io, record, key->key, version, - frame->error) || - (kind == TXN_RUNS_KIND_IDENTITY && !txn_runs_map_counted(frame->io, version, frame->error))) + frame->error)) return false; } } @@ -4719,17 +4870,41 @@ txn_runs_pushed_loading(txn_runs_pushed_runs_frame_0_t *frame, const HostStorage return true; } -/* txn_runs_pushed_spill links a storage's spill file, with its tuple-id file, and counts its versions at its transaction's entry. */ +/* + * txn_runs_pushed_spill links a storage's spill file, with its tuple-id file, + * and counts its versions at its transaction's entry; a fold's residual + * whose point isn't its storage's newest base's read point is passed over, + * and the startup records a free space map's spiller. + */ static bool txn_runs_pushed_spill(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind, const TxnRunsNames *names, const RepoRunFile *file) { + uint64 from = file->spilled.has_from ? file->spilled.from : 0; TxnSpillFile listing[2] = {{.xid = file->spilled.xid, .kind = kind, .number = file->spilled.number, - .len = file->len, .hash = file->hash}}; + .len = file->len, .hash = file->hash, .passed = file->spilled.has_passed ? file->spilled.passed : 0, + .from = from}}; size_t listing_len = 1; const RepoRunFile *tuples = txn_runs_tuple_ids(&frame->runs->files, file); txn_TupleU64KindU32OptionU32_t none = {0}; + if (from != 0) + { + bool some; + TxnBaseBase newest; + uint64 read = 0; + + if (!txn_state_newest(frame->io, &frame->loading.held, &some, &newest, frame->error)) + return false; + if (some) + { + read = newest.read; + txn_base_Base_drop(&newest); + } + if (from != read) + return true; + } + for (size_t k = 0; kind == TXN_RUNS_KIND_SPILLED_ROWS && k < frame->decoded.keys.len; k++) { const TxnRunsKey *key = &STD_VEC_AT(&frame->decoded.keys, TxnRunsKey, k); @@ -4771,8 +4946,10 @@ txn_runs_pushed_spill(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind, c if (!txn_spill_linked(&TXN_SPILL_STATE_HOME_OPS, frame->io, host_shared_Held_record(&frame->loading.held), &frame->loading.storage, listing, listing_len, false, none, frame->error)) return false; + if (kind == TXN_RUNS_KIND_SPILLED_FREE && !txn_spill_spilled_free(frame->io, file->spilled.xid, frame->error)) + return false; if (kind == TXN_RUNS_KIND_SPILLED_ROWS || kind == TXN_RUNS_KIND_SPILLED_ENTRIES || - kind == TXN_RUNS_KIND_SPILLED_VALUES) + kind == TXN_RUNS_KIND_SPILLED_VALUES || kind == TXN_RUNS_KIND_SPILLED_FREE) { HostXid xid = {file->spilled.xid}; bool has; @@ -4819,7 +4996,6 @@ txn_runs_pushed_row(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind, Std TxnMemtableVersionChange change = {.writer = txn_memtable_version_Record_writer(version), .op = &frame->op, .lock = txn_memtable_version_Record_lock(version), .shown = txn_memtable_version_Record_shown(version)}; txn_TupleRefSliceU8Usize_t row = {key, txn_memtable_version_Record_bytes(version).len}; - HostXid top = {change.writer.has_top ? change.writer.top : change.writer.xid}; bool counts; captures.has_tuple = kind == TXN_RUNS_KIND_ROWS && txn_runs_tid(&op, &captures.tuple); @@ -4837,15 +5013,8 @@ txn_runs_pushed_row(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind, Std change.has_lsn = txn_memtable_version_Record_lsn(version, &change.lsn); captures.change = &change; counts = frame->op.tag != TXN_MEMTABLE_OP_LOCK && frame->op.tag != TXN_MEMTABLE_OP_LOCK_ROW; - if (counts) - { - TxnStatusEntry entry; - - if (!txn_status_lookup(frame->io, top, &counts, &entry, frame->error)) - return false; - } if (counts && - !txn_status_redo_roomed(frame->io, &change.writer, &captures.has_roomed, &captures.roomed, frame->error)) + !txn_status_kept_roomed(frame->io, &change.writer, &captures.has_roomed, &captures.roomed, frame->error)) return false; if (!txn_memtable_table_prepare_Unit(frame->io, host_shared_Held_record(&frame->loading.held), &row, 1, NULL, 0, captures.has_tuple, txn_runs_pushed_row_closure_0, &captures, NULL, @@ -4868,6 +5037,8 @@ txn_runs_pushed_versions(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind twin = TXN_RUNS_KIND_SPILLED_ROWS; else if (kind == TXN_RUNS_KIND_ENTRIES) twin = TXN_RUNS_KIND_SPILLED_ENTRIES; + else if (kind == TXN_RUNS_KIND_FREE) + twin = TXN_RUNS_KIND_SPILLED_FREE; else if (kind != TXN_RUNS_KIND_VALUES) has_twin = false; for (size_t k = 0; k < frame->decoded.keys.len; k++) @@ -4895,18 +5066,16 @@ txn_runs_pushed_versions(txn_runs_pushed_runs_frame_0_t *frame, TxnRunsKind kind else if (kind == TXN_RUNS_KIND_FREE) { if (!txn_runs_pushed_back(&TXN_MEMTABLE_MAP_FREE_TREE_OPS, frame->io, record, key->key, version, - frame->error) || - !txn_runs_map_counted(frame->io, version, frame->error)) + frame->error)) return false; if (txn_memtable_version_Record_ender(version, &ender) && !txn_memtable_map_ended(frame->io, record, - txn_memtable_map_taken(txn_memtable_version_Record_op(version)), ender, - frame->error)) + txn_memtable_map_taken(txn_memtable_version_Record_op(version)), + txn_memtable_version_Record_writer(version).xid, ender, frame->error)) return false; } else if (!txn_runs_pushed_back(&TXN_MEMTABLE_MAP_VALUES_TREE_OPS, frame->io, record, key->key, version, - frame->error) || - !txn_runs_map_counted(frame->io, version, frame->error)) + frame->error)) return false; } } @@ -4959,7 +5128,8 @@ txn_runs_pushed_file(txn_runs_pushed_runs_frame_0_t *frame, const RepoRunFile *f if (!txn_runs_decode(frame->bytes.ptr, frame->bytes.len, &names, file->hash, &frame->decoded, frame->error)) return false; frame->decoded_live = true; - if (kind == TXN_RUNS_KIND_SPILLED_CHUNKS || kind == TXN_RUNS_KIND_CHUNKS || kind == TXN_RUNS_KIND_IDENTITY) + if (kind == TXN_RUNS_KIND_SPILLED_CHUNKS || kind == TXN_RUNS_KIND_CHUNKS || kind == TXN_RUNS_KIND_IDENTITY || + kind == TXN_RUNS_KIND_SPILLED_IDENTITY) return txn_runs_pushed_branch(frame, kind, file); if (!txn_runs_pushed_loading(frame, &storage)) return false; @@ -6341,13 +6511,15 @@ txn_runs_tests_first_prepared(void *captures, HostIoIo *io, TxnMemtableTablePrep const txn_runs_tests_loaded_captures_t *c = captures; StdBytes values[2] = {{c->free->ptr, c->free->len}, {(const uint8 *) "value", 5}}; HostRow tid = {5}; + TxnStatusRoomed unroomed = {0}; for (size_t take = 0; take < lengthof(values); take++) { TxnMemtableMapTaken taken; - if (!txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, take, c->change->writer, false, 0, - false, NULL, 0, txn_runs_tests_put_step, &values[take], NULL, + if (!txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, take, c->change->writer, false, + unroomed, false, 0, + false, NULL, 0, NULL, txn_runs_tests_put_step, &values[take], NULL, txn_runs_tests_nothing, NULL, NULL, &taken, error)) return false; } @@ -6869,9 +7041,11 @@ txn_runs_tests_cut_prepared(void *captures, HostIoIo *io, TxnMemtableTablePrepar { txn_runs_tests_cut_captures_t *c = captures; TxnMemtableMapTaken taken; + TxnStatusRoomed unroomed = {0}; - return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, c->change->writer, false, 0, false, - NULL, 0, txn_runs_tests_put_step, &c->free, NULL, txn_runs_tests_nothing, NULL, + return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, c->change->writer, false, unroomed, + false, 0, false, + NULL, 0, NULL, txn_runs_tests_put_step, &c->free, NULL, txn_runs_tests_nothing, NULL, NULL, &taken, error) && txn_runs_tests_rowed(io, prepared, c->tuple, c->change, error); } @@ -7041,11 +7215,11 @@ test_txn_runs_tests_a_start_refuses_a_free_space_run_whose_counter_isnt_a_tuple_ { record.ptr, record.len }; - if (!txn_runs_Writing_key(writing, io, key, sizeof(key), records, 1, &error)) + if (!txn_runs_Writing_key(writing, HOST_IO_IO_IO_DOORS(io), key, sizeof(key), records, 1, &error)) STD_RESULT_EXPECT_FAILED(2244, 90, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); std_vec_free_Checked(&record, 1, 1); - if (!txn_runs_Writing_end(writing, io, NULL, 0, &some, &file, &error)) + if (!txn_runs_Writing_end(writing, HOST_IO_IO_IO_DOORS(io), NULL, 0, &some, &file, &error)) STD_RESULT_EXPECT_FAILED(2245, 46, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); std_vec_new(&runs.files, sizeof(RepoRunFile), _Alignof(RepoRunFile)); @@ -8598,11 +8772,11 @@ txn_runs_tests_entry_version(const TxnMemtableOp *op) return out; } -/* txn_runs_tests_entry_put returns an entries run's put or add of an entry, naming a tuple id with heap bytes. */ +/* txn_runs_tests_entry_put returns a run's put or add of a value, naming a tuple id with heap bytes. */ static TxnMemtableOp -txn_runs_tests_entry_put(TxnMemtableOpTag tag, uint64 tuple, uint32 heap_bytes) +txn_runs_tests_entry_put(TxnMemtableOpTag tag, const char *value, uint64 tuple, uint32 heap_bytes) { - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "entry", 5); + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) value, strlen(value)); op.tag = tag; op.u.put.tuple.f0 = tuple; @@ -8641,12 +8815,12 @@ test_txn_runs_tests_an_entries_run_reads_its_entries_and_refuses_heap_bytes_tupl entries.kind = TXN_RUNS_KIND_ENTRIES; entries.relation = 16390; - reads[0] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, 0, 0); + reads[0] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "entry", 0, 0); reads[1] = (TxnMemtableOp) { .tag = TXN_MEMTABLE_OP_DELETE }; - reads[2] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_ADD, copy, 0); + reads[2] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_ADD, "entry", copy, 0); reads[3] = (TxnMemtableOp) { .tag = TXN_MEMTABLE_OP_REMOVE, .u.remove.f0.f0 = copy @@ -8661,9 +8835,9 @@ test_txn_runs_tests_an_entries_run_reads_its_entries_and_refuses_heap_bytes_tupl "decoded.keys.len() == 1 && decoded.keys[0].versions.len() == 4"); txn_runs_Decoded_drop(&decoded); std_vec_free_Checked(&good, 1, 1); - refuses[0] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, 0, 32); - refuses[1] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_ADD, copy, 32); - refuses[2] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, copy, 0); + refuses[0] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "entry", 0, 32); + refuses[1] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_ADD, "entry", copy, 32); + refuses[2] = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "entry", copy, 0); refuses[3] = (TxnMemtableOp) { .tag = TXN_MEMTABLE_OP_MOVE, .u.move.tag = TXN_MEMTABLE_SUCCESSOR_KEY @@ -8809,7 +8983,7 @@ test_txn_runs_tests_a_runs_decoder_takes_a_rows_ender_only_on_a_transactions_tak TxnRunsNames values = TXN_RUNS_TESTS_NAMES; TxnRunsNames cases_names[5]; uint8 block[4]; - TxnMemtableOp op = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, 0, 0); + TxnMemtableOp op = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "entry", 0, 0); TxnMemtableVersionWriter writer = {.xid = 7}; TxnMemtableVersionShown shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; StdVec untaken; @@ -8886,9 +9060,11 @@ txn_runs_tests_ender_prepared(void *captures, HostIoIo *io, TxnMemtableTablePrep TxnMemtableVersionWriter writer = {.xid = 7}; HostRow none = {0}; TxnMemtableMapTaken taken; + TxnStatusRoomed unroomed = {0}; - return txn_memtable_map_took_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, writer, false, 0, false, none, 0, false, - NULL, 0, txn_runs_tests_put_step, captures, NULL, txn_runs_tests_row_one, NULL, + return txn_memtable_map_took_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, writer, false, unroomed, false, 0, false, + none, 0, false, + NULL, 0, NULL, txn_runs_tests_put_step, captures, NULL, txn_runs_tests_row_one, NULL, NULL, txn_runs_tests_nothing, NULL, NULL, &taken, error); } @@ -8929,7 +9105,7 @@ test_txn_runs_tests_a_takes_ender_loads_back_from_its_free_space_run(void) HostRow one = {1}; if (!txn_memtable_map_prepare_Unit(io, &take, 1, txn_runs_tests_ender_prepared, &value, NULL, &error) || - !txn_memtable_map_ended(io, host_shared_Held_record(&held), one, 9, &error)) + !txn_memtable_map_ended(io, host_shared_Held_record(&held), one, 7, 9, &error)) STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); txn_state_moved(host_shared_Held_record(&held), 1, (UINT64CONST(4) << 16) | 1); @@ -9015,7 +9191,7 @@ test_txn_runs_tests_an_index_storages_loaded_entries_stay_an_entries_run_through txn_runs_tests_repository(io, 9490, __LINE__, 0, __LINE__, 0, &repo, &reading); { HostSharedHeld held; - TxnMemtableOp op = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, 0, 0); + TxnMemtableOp op = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "entry", 0, 0); TxnMemtableVersionChange change = {.writer = {.xid = 7}, .op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}, .has_lsn = true, .lsn = 0x0a28}; TxnMemtableVersionHandle handle; @@ -9269,17 +9445,17 @@ typedef struct txn_runs_tests_index_entry_t } txn_runs_tests_index_entry_t; /* - * txn_runs_tests_spill_raw returns a spill's file built by hand: its - * header's bytes, each key frame's record, an index of the key frames' - * numbers, each entry naming that frame's place with a key given, and its - * count of versions. + * txn_runs_tests_spill_raw returns a spill's file of its names built by + * hand: its header's bytes, each key frame's record, an index of the key + * frames' numbers, each entry naming that frame's place with a key given, and + * its count of versions. */ static StdVec -txn_runs_tests_spill_raw(const StdVec *records, size_t records_len, const txn_runs_tests_index_entry_t *index, - size_t index_len, uint64 count) +txn_runs_tests_spill_raw(const TxnRunsNames *names, const StdVec *records, size_t records_len, + const txn_runs_tests_index_entry_t *index, size_t index_len, uint64 count) { StdVec out; - StdVec header = txn_runs_tests_header(&TXN_RUNS_TESTS_SPILLED); + StdVec header = txn_runs_tests_header(names); StdVec entries; uint64 places[2]; uint64 at; @@ -9340,7 +9516,7 @@ test_txn_runs_tests_a_spills_decoder_refuses_another_writer_and_a_wrong_index(vo frames[0] = txn_runs_tests_keyed((const uint8 *) "a", 1, &versions[0], 1); frames[1] = txn_runs_tests_keyed((const uint8 *) "b", 1, &versions[1], 1); frames[2] = txn_runs_tests_keyed((const uint8 *) "b", 1, &versions[2], 1); - good = txn_runs_tests_spill_raw(frames, 2, both, lengthof(both), 2); + good = txn_runs_tests_spill_raw(&TXN_RUNS_TESTS_SPILLED, frames, 2, both, lengthof(both), 2); if (!txn_runs_decode(good.ptr, good.len, &TXN_RUNS_TESTS_SPILLED, txn_runs_tests_sum_spill(&good), &decoded, &error)) STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, @@ -9350,11 +9526,11 @@ test_txn_runs_tests_a_spills_decoder_refuses_another_writer_and_a_wrong_index(vo { StdVec others[2] = {frames[0], frames[2]}; - refused[0] = txn_runs_tests_spill_raw(others, 2, first_only, lengthof(first_only), 2); + refused[0] = txn_runs_tests_spill_raw(&TXN_RUNS_TESTS_SPILLED, others, 2, first_only, lengthof(first_only), 2); } - refused[1] = txn_runs_tests_spill_raw(frames, 2, wrong_key, lengthof(wrong_key), 2); - refused[2] = txn_runs_tests_spill_raw(frames, 2, reversed, lengthof(reversed), 2); - refused[3] = txn_runs_tests_spill_raw(frames, 2, NULL, 0, 2); + refused[1] = txn_runs_tests_spill_raw(&TXN_RUNS_TESTS_SPILLED, frames, 2, wrong_key, lengthof(wrong_key), 2); + refused[2] = txn_runs_tests_spill_raw(&TXN_RUNS_TESTS_SPILLED, frames, 2, reversed, lengthof(reversed), 2); + refused[3] = txn_runs_tests_spill_raw(&TXN_RUNS_TESTS_SPILLED, frames, 2, NULL, 0, 2); for (size_t at = 0; at < lengthof(refused); at++) { STD_ASSERT(__LINE__, 0, @@ -9372,6 +9548,78 @@ test_txn_runs_tests_a_spills_decoder_refuses_another_writer_and_a_wrong_index(vo } } +/* txn_runs_tests_identity_version returns an identity spill's version of an op by a writer. */ +static StdVec +txn_runs_tests_identity_version(uint64 xid, const TxnMemtableOp *op) +{ + TxnMemtableVersionWriter writer = {.xid = xid}; + TxnMemtableVersionShown shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + StdVec out; + + std_vec_new(&out, 1, 1); + txn_memtable_version_encode(&writer, op, TXN_MEMTABLE_LOCK_MODE_UPDATE, shown, true, 9, &out); + return out; +} + +/* + * Bugs row 291's run kind 13: an identity spill reads back its puts naming + * no tuple id and its removals, and the decoder refuses as XX001 another + * transaction's version, a put naming a tuple id, and a put with heap bytes. + */ +void +test_txn_runs_tests_an_identity_spill_reads_its_pairs_and_refuses_another_writer_tuple_ids_and_heap_bytes(void) +{ + static const txn_runs_tests_index_entry_t first[1] = {{0, "a"}}; + TxnRunsNames identity = TXN_RUNS_TESTS_SPILLED; + TxnMemtableOp reads[2] = {{.tag = TXN_MEMTABLE_OP_DELETE}, txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "pair", 0, 0)}; + TxnMemtableOp refuses[3] = {txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "pair", 0, 0), + txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "pair", 5, 0), + txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "pair", 0, 32)}; + static const uint64 writers[3] = {743, 742, 742}; + StdVec read[2]; + StdVec frame; + StdVec good; + TxnRunsDecoded decoded; + HostError error; + + identity.kind = TXN_RUNS_KIND_SPILLED_IDENTITY; + identity.tablespace = 0; + identity.relation = 1; + for (size_t k = 0; k < lengthof(reads); k++) + read[k] = txn_runs_tests_identity_version(742, &reads[k]); + frame = txn_runs_tests_keyed((const uint8 *) "a", 1, read, lengthof(read)); + good = txn_runs_tests_spill_raw(&identity, &frame, 1, first, lengthof(first), 2); + if (!txn_runs_decode(good.ptr, good.len, &identity, txn_runs_tests_sum_spill(&good), &decoded, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, decoded.keys.len == 1 && STD_VEC_AT(&decoded.keys, TxnRunsKey, 0).versions.len == 2, + "decoded.keys[0].versions.len() == 2"); + txn_runs_Decoded_drop(&decoded); + std_vec_free_Checked(&good, 1, 1); + std_vec_free_Checked(&frame, 1, 1); + for (size_t at = 0; at < lengthof(refuses); at++) + { + StdVec one = txn_runs_tests_identity_version(writers[at], &refuses[at]); + StdVec keyed = txn_runs_tests_keyed((const uint8 *) "a", 1, &one, 1); + StdVec bytes = txn_runs_tests_spill_raw(&identity, &keyed, 1, first, lengthof(first), 1); + + STD_ASSERT(__LINE__, 0, + !txn_runs_decode(bytes.ptr, bytes.len, &identity, txn_runs_tests_sum_spill(&bytes), &decoded, + &error), + "the identity spill decoder refuses a case"); + txn_runs_tests_sqlstate_is(__LINE__, 0, &error); + std_vec_free_Checked(&bytes, 1, 1); + std_vec_free_Checked(&keyed, 1, 1); + std_vec_free_Checked(&one, 1, 1); + txn_memtable_Op_drop(&refuses[at]); + } + for (size_t k = 0; k < lengthof(reads); k++) + { + std_vec_free_Checked(&read[k], 1, 1); + txn_memtable_Op_drop(&reads[k]); + } +} + /* txn_runs_tests_spill_files counts a repository's spill files in pg_default, leaving out its spills' manifests. */ static size_t txn_runs_tests_spill_files(HostIoIo *io, uint32 repository) @@ -9539,7 +9787,7 @@ test_txn_runs_tests_a_checkpoint_names_a_running_spill_and_a_start_links_it_back STD_ASSERT(__LINE__, 0, some && entry.count == 3, "status::lookup(io, mine).unwrap().map(|e| e.count) == Some(3)"); STD_ASSERT_EQ_U64(__LINE__, 0, txn_runs_tests_spill_files(io, 9748), 2); simc_abort(mine); - if (!txn_spill_aborted(io, mine.f0, &storage, 1, &error)) + if (!txn_spill_aborted(io, mine.f0, &storage, 1, 0, &error)) STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); for (size_t k = 0; k < lengthof(later); k++) @@ -9561,4 +9809,971 @@ test_txn_runs_tests_a_checkpoint_names_a_running_spill_and_a_start_links_it_back simc_TestIo_drop(&test_io); } +/* txn_runs_tests_entry_prepared is the entries spill case's prepare: the entry's change at its key, with no tuple id. */ +static bool +txn_runs_tests_entry_prepared(void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, + HostError *error) +{ + TxnStatusRoomed roomed = {0}; + TxnMemtableVersionHandle handle; + + return txn_memtable_table_finish(HOST_IO_IO_TAKES_SHORT(io), prepared, captures, false, roomed, &handle, error); +} + +/* + * Run kind 9: an index's storage spills its transaction's entries as an + * entries spill, holding no tuple ids, which a checkpoint names beside its + * runs and a start links back, the entries read again and counted at their + * writer's entry, and the index's state made from the spill's header with + * its table and index (bugs row 161). + */ +void +test_txn_runs_tests_an_index_storages_entries_spill_and_load_back_as_entries(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostStorage storage = {.tablespace = HOST_PG_DEFAULT, .database = 9749, .relation = 16390}; + TxnStateMade made = {.key = {.tag = TXN_BASE_KEY_NONE}, .table = 16384, .has_index = true, .index = 16390}; + TxnInsertPage page = {.bytes = 8160, .most = 291}; + TxnStatusOutcome unowned = {.tag = TXN_STATUS_OUTCOME_UNOWNED}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostXid mine; + StdArcInner *manifest; + TxnStatusRoomed roomed; + TxnStatusEntry entry; + TxnBaseBase base; + ChunksErrorError failed; + HostError error; + bool took; + bool some; + bool spilled; + + txn_runs_tests_repository(io, 9749, __LINE__, 0, __LINE__, 0, &repo, &reading); + mine = simc_begin(); + if (!txn_state_state(io, &repo, &reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, true, &made, &held, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (uint64 number = 1; number <= 3; number++) + { + TxnMemtableOp op = txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "entry", 0, 0); + TxnMemtableVersionChange change = {.writer = {.xid = mine.f0}, .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}, + .has_lsn = true, .lsn = 0x0a00 + number}; + txn_TupleRefSliceU8Usize_t rows[1]; + uint8 key[8]; + + std_u64_to_be_bytes(number, key); + rows[0].f0.ptr = key; + rows[0].f0.len = sizeof(key); + rows[0].f1 = txn_memtable_version_encoded(&change.writer, change.op, change.shown, change.has_lsn, change.lsn); + if (!txn_memtable_table_prepare_Unit(io, host_shared_Held_record(&held), rows, 1, NULL, 0, false, + txn_runs_tests_entry_prepared, &change, NULL, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + } + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine.f0, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, spilled, "crate::spill::spilled(io, &storage, Of::Rows, mine).unwrap()"); + host_shared_Held_drop(&held); + if (!txn_status_take(io, mine, unowned, &roomed, &took, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_checkpointed(UINT64CONST(0x100000a27)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000a28), &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!repo_Repo_manifest(&repo, io, &manifest, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + { + const RepoManifest *named = std_arc_Deref_deref(manifest, _Alignof(RepoManifest)); + size_t found = 0; + bool entries = true; + + for (size_t k = 0; named->has_runs && k < named->runs.files.len; k++) + { + const RepoRunFile *file = &STD_VEC_AT(&named->runs.files, RepoRunFile, k); + + if (!file->has_spilled) + continue; + entries = entries && file->kind == TXN_RUNS_KIND_SPILLED_ENTRIES && file->spilled.xid == mine.f0; + found++; + } + STD_ASSERT(__LINE__, 0, found == 1 && entries, "spilled == [(SpilledEntries, mine)]"); + } + std_arc_drop(manifest, sizeof(RepoManifest), _Alignof(RepoManifest), txn_runs_tests_Manifest_drop_glue); + if (!txn_state_unlinked(io, &storage, &error) || !txn_runs_loaded(io, &page, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + held = txn_runs_tests_stated(io, &repo, &reading, &storage, __LINE__, 0); + if (!txn_state_base_record(io, &held, &some, &base, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, some && base.table == 16384 && base.has_index && base.index == 16390, + "(base.table, base.index) == (16384, Some(16390))"); + if (some) + txn_base_Base_drop(&base); + { + HostDeltaHold hold; + + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (uint64 number = 1; number <= 3; number++) + { + StdVec versions; + uint8 key[8]; + + std_u64_to_be_bytes(number, key); + if (!txn_memtable_table_versions(io, &hold, host_shared_Held_record(&held), key, sizeof(key), &versions, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ_U64(__LINE__, 0, versions.len, 1); + txn_memtable_version_versions_drop(&versions, 0); + } + host_delta_Hold_drop(&hold); + } + if (!txn_status_lookup(io, mine, &some, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, some && entry.count == 3, "status::lookup(io, mine).unwrap().map(|e| e.count) == Some(3)"); + simc_abort(mine); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_runs_tests_pushed_rows pushes puts of "v" by a writer at keys of numbers, each naming its number as its tuple id. */ +static void +txn_runs_tests_pushed_rows(HostIoIo *io, const RepoRepo *repo, const HostStorage *storage, uint64 xid, uint64 first, + uint64 last) +{ + HostSharedReader reading; + HostSharedHeld held; + HostError error; + + if (!host_shared_read(io, &reading, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + held = txn_runs_tests_stated(io, repo, &reading, storage, __LINE__, 0); + for (uint64 number = first; number <= last; number++) + { + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = {.xid = xid}, .op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, + .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + uint8 key[8]; + + op.u.put.tuple.f0 = number; + std_u64_to_be_bytes(number, key); + if (!txn_memtable_table_push(io, host_shared_Held_record(&held), key, sizeof(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); +} + +/* txn_runs_tests_merge_t is the merge source case's actors' captures: the storage, the writer, and what the reader found. */ +typedef struct txn_runs_tests_merge_t +{ + HostStorage storage; + uint64 xid; + bool found; + size_t versions; +} txn_runs_tests_merge_t; + +/* txn_runs_tests_merge_reader reads key 1's versions in a state of its own, after the spill's listing. */ +static bool +txn_runs_tests_merge_reader(void *captures, HostIoIo *io, HostError *error) +{ + txn_runs_tests_merge_t *c = captures; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostDeltaHold hold; + StdVec versions; + ChunksErrorError failed; + uint8 key[8]; + bool ok; + + if (!repo_Repo_open(io, c->storage.database, HOST_PG_DEFAULT, &repo, &failed)) + { + *error = txn_base_storage(&failed); + return false; + } + if (!host_shared_read(io, &reading, error)) + { + repo_Repo_drop(&repo); + return false; + } + ok = txn_state_state(io, &repo, &reading, &c->storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &held, + error); + if (ok) + { + ok = host_delta_hold(io, &hold, error); + if (ok) + { + std_u64_to_be_bytes(1, key); + ok = txn_memtable_table_versions(io, &hold, host_shared_Held_record(&held), key, sizeof(key), &versions, + error); + if (ok) + { + c->found = true; + c->versions = versions.len; + txn_memtable_version_versions_drop(&versions, 0); + } + host_delta_Hold_drop(&hold); + } + host_shared_Held_drop(&held); + } + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* txn_runs_tests_merge_merger spills the writer's rows again, merging its spills, then aborts the writer. */ +static bool +txn_runs_tests_merge_merger(void *captures, HostIoIo *io, HostError *error) +{ + txn_runs_tests_merge_t *c = captures; + HostXid xid = {c->xid}; + bool spilled; + + if (!txn_spill_spilled(io, &c->storage, TXN_WRITTEN_OF_ROWS, c->xid, &spilled, error)) + return false; + STD_ASSERT(__LINE__, 0, spilled, "crate::spill::spilled(io, &storage, Of::Rows, mine)?"); + simc_abort(xid); + return txn_spill_aborted(io, c->xid, &c->storage, 1, 0, error); +} + +/* txn_runs_tests_merge_checkpointer runs two checkpoints. */ +static bool +txn_runs_tests_merge_checkpointer(void *captures, HostIoIo *io, HostError *error) +{ + static const HostLsn redos[2] = {UINT64CONST(0x100000e28), UINT64CONST(0x100000f28)}; + + for (size_t k = 0; k < lengthof(redos); k++) + { + simc_checkpointed(redos[k] - 1); + if (!txn_runs_checkpoint(io, redos[k], error)) + return false; + } + return true; +} + +/* txn_runs_tests_all_returned asserts at its line that a run ended with every actor's body returning Ok. */ +static void +txn_runs_tests_all_returned(uint32 line, const SimcReport *report) +{ + bool all = report->outcome.tag == SIMC_OUTCOME_ENDED; + + for (size_t k = 0; all && k < report->outcome.u.ended.len; k++) + { + const SimcEnding *ending = &STD_VEC_AT(&report->outcome.u.ended, simc_TupleStringEnding_t, k).f1; + + all = ending->tag == SIMC_ENDING_RETURNED && !ending->is_err; + } + STD_ASSERT_EQ(line, 0, all, report, simc_Report_Debug_fmt, report, simc_Report_Debug_fmt); +} + +/* + * Bugs row 124's M3 for a merge's sources: a reader that listed a + * transaction's first spill of a storage, paused before it reads it, while + * the transaction's next spill merges both into one file and the transaction + * aborts, and two checkpoints run, the second of which finds the source + * unnamed past its listing, reads the source still, since the reader began + * before the listing's epoch. + */ +void +test_txn_runs_tests_a_reader_that_listed_a_merges_source_reads_it_across_two_checkpoints(void) +{ + static const SimcStep steps[3] = {{"merger", 6, PG_UINT64_MAX}, {"checkpointer", 12, PG_UINT64_MAX}, + {"reader", 6, PG_UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_runs_tests_merge_t c = {.storage = {.tablespace = HOST_PG_DEFAULT, .database = 9750, .relation = 16384}}; + SimcActor reader = {txn_runs_tests_merge_reader, &c}; + SimcActor merger = {txn_runs_tests_merge_merger, &c}; + SimcActor checkpointer = {txn_runs_tests_merge_checkpointer, NULL}; + RepoRepo repo; + SimcRun run; + SimcReport report; + ChunksErrorError failed; + HostError error; + bool spilled; + + if (!repo_Repo_create(io, 9750, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + c.xid = simc_begin().f0; + txn_runs_tests_pushed_rows(io, &repo, &c.storage, c.xid, 1, 3); + if (!txn_spill_spilled(io, &c.storage, TXN_WRITTEN_OF_ROWS, c.xid, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, spilled, "crate::spill::spilled(io, &storage, Of::Rows, mine).unwrap()"); + txn_runs_tests_pushed_rows(io, &repo, &c.storage, c.xid, 4, 6); + run = simc_Run_new(1); + simc_Run_actor(&run, "reader", 6, reader); + simc_Run_actor(&run, "merger", 6, merger); + simc_Run_actor(&run, "checkpointer", 12, checkpointer); + simc_Run_until(&run, "reader", 6, "doltgres-spill-listed"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_runs_tests_all_returned(__LINE__, &report); + STD_ASSERT(__LINE__, 0, c.found && c.versions == 1, "*found.lock().unwrap() == Some(1)"); + simc_Report_drop(&report); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* txn_runs_tests_part_holder writes a transaction's rows, touched first, then pauses holding nothing and aborts. */ +static bool +txn_runs_tests_part_holder(void *captures, HostIoIo *io, HostError *error) +{ + const HostStorage *storage = captures; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + ChunksErrorError failed; + HostXid xid; + bool ok = true; + + if (!repo_Repo_open(io, storage->database, HOST_PG_DEFAULT, &repo, &failed)) + { + *error = txn_base_storage(&failed); + return false; + } + if (!host_shared_read(io, &reading, error)) + { + repo_Repo_drop(&repo); + return false; + } + if (!txn_state_state(io, &repo, &reading, storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &held, error)) + { + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return false; + } + xid = simc_begin(); + for (uint64 number = 1; ok && number <= 3; number++) + { + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = {.xid = xid.f0}, .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + uint8 key[8]; + uint64 touched; + + op.u.put.tuple.f0 = number; + std_u64_to_be_bytes(number, key); + ok = txn_written_touched(io, xid.f0, storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), + txn_memtable_version_encoded(&change.writer, change.op, change.shown, false, 0), + &touched, error) && + txn_memtable_table_push(io, host_shared_Held_record(&held), key, sizeof(key), &change, &handle, error); + txn_memtable_Op_drop(&op); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + if (ok && host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-held", error)) + { + simc_abort(xid); + ok = txn_written_aborted(io, 0, error); + } + else + ok = false; + repo_Repo_drop(&repo); + return ok; +} + +/* txn_runs_tests_part_helper relieves the delta by spilling a holder's versions, asserting it spilled. */ +static bool +txn_runs_tests_part_helper(void *captures, HostIoIo *io, HostError *error) +{ + bool relieved; + + if (!txn_spill_relieved(io, &relieved, error)) + return false; + STD_ASSERT(__LINE__, 0, relieved, "crate::spill::relieved(io)?"); + return true; +} + +/* txn_runs_tests_part_checkpointer runs three checkpoints. */ +static bool +txn_runs_tests_part_checkpointer(void *captures, HostIoIo *io, HostError *error) +{ + static const HostLsn redos[3] = {UINT64CONST(0x100001028), UINT64CONST(0x100001128), UINT64CONST(0x100001228)}; + + for (size_t k = 0; k < lengthof(redos); k++) + { + simc_checkpointed(redos[k] - 1); + if (!txn_runs_checkpoint(io, redos[k], error)) + return false; + } + return true; +} + +/* + * Bugs row 144's abort during a helper's spill: a holder's transaction aborts + * while a helper that spills its versions is held between its removal and + * its link, so the helper links a part of an aborted transaction after its + * take-back; the next checkpoint takes the aborted transaction's entry out + * of the chain, unnamed, and a later one, past every reader, removes its + * files. + */ +void +test_txn_runs_tests_a_helpers_part_linked_after_its_transactions_abort_leaves_with_its_files(void) +{ + static const SimcStep steps[3] = {{"holder", 6, PG_UINT64_MAX}, {"helper", 6, PG_UINT64_MAX}, + {"checkpointer", 12, PG_UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostStorage storage = {.tablespace = HOST_PG_DEFAULT, .database = 9751, .relation = 16384}; + SimcActor holder = {txn_runs_tests_part_holder, &storage}; + SimcActor helper = {txn_runs_tests_part_helper, NULL}; + SimcActor checkpointer = {txn_runs_tests_part_checkpointer, NULL}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + SimcRun run; + SimcReport report; + txn_TupleVecFileU64_t listed; + StdPathBuf dir; + StdVec names; + ChunksErrorError failed; + HostError error; + bool some; + + if (!repo_Repo_create(io, 9751, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + repo_Repo_drop(&repo); + run = simc_Run_new(1); + simc_Run_actor(&run, "holder", 6, holder); + simc_Run_actor(&run, "helper", 6, helper); + simc_Run_actor(&run, "checkpointer", 12, checkpointer); + simc_Run_until(&run, "holder", 6, "doltgres-spill-case-held"); + simc_Run_until(&run, "helper", 6, "doltgres-spill-linking"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_runs_tests_all_returned(__LINE__, &report); + simc_Report_drop(&report); + if (!repo_Repo_open(io, 9751, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!host_shared_read(io, &reading, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + held = txn_runs_tests_stated(io, &repo, &reading, &storage, __LINE__, 0); + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, host_shared_Held_record(&held), &storage, &listed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, listed.f0.len == 0, "the aborted part leaves the manifest"); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + txn_runs_spill_directory(9751, HOST_PG_DEFAULT, &dir); + if (!host_list_dir_if_there(HOST_IO_IO_READ_DOORS(io), dir.bytes.ptr, dir.bytes.len, &some, &names, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, !some || names.len == 0, "its files go"); + if (some) + txn_runs_names_drop(&names); + std_vec_free_Checked(&dir.bytes, 1, 1); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + +/* + * PLAN item 10's run kind 10: a database's identity tree's run reads back its + * puts naming no tuple id and its removals, and the decoder refuses as XX001 + * a put with heap bytes, a put naming a tuple id, and a lock. + */ +void +test_txn_runs_tests_an_identity_run_reads_its_pairs_and_refuses_heap_bytes_tuple_ids_and_other_ops(void) +{ + TxnRunsNames identity = TXN_RUNS_TESTS_NAMES; + TxnMemtableOp reads[2] = {{.tag = TXN_MEMTABLE_OP_DELETE}, txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "pair", 0, 0)}; + TxnMemtableOp refuses[3] = {txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "pair", 0, 32), txn_runs_tests_entry_put(TXN_MEMTABLE_OP_PUT, "pair", 5, 0), + {.tag = TXN_MEMTABLE_OP_LOCK, .u.lock = TXN_MEMTABLE_LOCK_MODE_SHARE}}; + StdVec read[2]; + StdVec good; + TxnRunsDecoded decoded; + HostError error; + + identity.kind = TXN_RUNS_KIND_IDENTITY; + identity.tablespace = 0; + identity.relation = 0; + for (size_t k = 0; k < lengthof(reads); k++) + read[k] = txn_runs_tests_entry_version(&reads[k]); + good = txn_runs_tests_entries_file(&identity, read, lengthof(read)); + if (!txn_runs_decode(good.ptr, good.len, &identity, txn_runs_tests_sum(&good), &decoded, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, decoded.keys.len == 1 && STD_VEC_AT(&decoded.keys, TxnRunsKey, 0).versions.len == 2, + "decoded.keys[0].versions.len() == 2"); + txn_runs_Decoded_drop(&decoded); + std_vec_free_Checked(&good, 1, 1); + for (size_t at = 0; at < lengthof(refuses); at++) + { + StdVec one = txn_runs_tests_entry_version(&refuses[at]); + StdVec bytes = txn_runs_tests_entries_file(&identity, &one, 1); + + STD_ASSERT(__LINE__, 0, + !txn_runs_decode(bytes.ptr, bytes.len, &identity, txn_runs_tests_sum(&bytes), &decoded, &error), + "the identity run decoder refuses a case"); + txn_runs_tests_sqlstate_is(__LINE__, 0, &error); + std_vec_free_Checked(&bytes, 1, 1); + std_vec_free_Checked(&one, 1, 1); + txn_memtable_Op_drop(&refuses[at]); + } + for (size_t k = 0; k < lengthof(reads); k++) + { + std_vec_free_Checked(&read[k], 1, 1); + txn_memtable_Op_drop(&reads[k]); + } +} + +/* txn_runs_tests_relation_keyed is the identity case's keyed: a relation's key by its id. */ +static void +txn_runs_tests_relation_keyed(const void *captures, uint64 id, StdVec *result) +{ + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, id, result); +} + +/* txn_runs_tests_identity_t is identity_checkpointed's answer: the pair's object id, whether the loaded delta holds an identity version, and what its two keys find. */ +typedef struct txn_runs_tests_identity_t +{ + uint64 object; + bool held; + bool found; + StdVec by_oid; + uint32 number; +} txn_runs_tests_identity_t; + +/* + * txn_runs_tests_identity_checkpointed records a relation's pair in a + * transaction committed at an end, checkpoints, then drops the working + * state's delta and loads its runs back as a start does, and returns the + * pair's object id, whether the loaded delta holds any identity version, and + * what a reader then finds by the pair's two keys. + */ +static txn_runs_tests_identity_t +txn_runs_tests_identity_checkpointed(HostIoIo *io, uint32 number, HostLsn end) +{ + TxnInsertPage page = {.bytes = 8160, .most = 291}; + TxnStatusOutcome committed = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = end}; + txn_runs_tests_identity_t answer = {0}; + HostIoWaits waits = host_io_Waits_testing(); + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld deltas; + HostXid sub; + HostXid top; + TxnMemtableVersionWriter writer = {0}; + TxnStatusRoomed roomed; + TxnStatusEntry entry; + StdVec numbered; + StdVec keyed; + StdVec chains; + StdVec found; + ChunksErrorError failed; + HostError error; + bool some; + bool settled; + + simc_end(false); + if (!repo_Repo_create(io, number, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + repo_Repo_drop(&repo); + if (!host_current_writer(HOST_IO_IO_IO_DOORS(io), &sub, &top, &error) || + !txn_status_roomed(io, top, &roomed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + writer.xid = sub.f0; + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, 16384, &numbered); + { + TxnIdentityDatabase tree; + + repo = txn_runs_tests_reopened(io, number); + if (!host_shared_read(io, &reading, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + tree.f0 = &repo; + tree.f1 = &reading; + tree.f2 = number; + if (!txn_identity_derived(io, &waits, tree, writer, roomed, TXN_IDENTITY_KIND_RELATION, 5, + (const uint8 *) "t", 1, numbered.ptr, numbered.len, 16384, + txn_runs_tests_relation_keyed, NULL, NULL, &answer.object, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + } + simc_end(true); + if (!txn_written_ended(io, &error) || !txn_status_lookup(io, top, &some, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!some) + STD_OPTION_UNWRAP_FAILED(__LINE__, 0); + if (!txn_status_settled(io, top, entry.outcome, committed, &settled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, settled, "status::settled(io, top, entry.outcome, Committed(end)).unwrap()"); + simc_checkpointed(UINT64CONST(0x100000a27)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000a28), &error) || + !txn_memtable_branch_dropped(io, number, &error) || !txn_runs_loaded(io, &page, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + repo = txn_runs_tests_reopened(io, number); + if (!host_shared_read(io, &reading, &error) || + !txn_memtable_branch_deltas(io, number, HOST_PG_DEFAULT, &deltas, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + chains = txn_runs_tests_chains(&TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS, io, host_shared_Held_record(&deltas)); + answer.held = chains.len != 0; + txn_runs_tests_chains_free(&chains); + { + TxnIdentityDatabase tree = {&repo, &reading, number}; + + if (!txn_identity_looked_up(io, &waits, tree, numbered.ptr, numbered.len, &answer.found, &answer.by_oid, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_identity_by_id(TXN_IDENTITY_KIND_RELATION, answer.object, &keyed); + if (!txn_identity_looked_up(io, &waits, tree, keyed.ptr, keyed.len, &some, &found, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!some) + STD_OPTION_UNWRAP_FAILED(__LINE__, 0); + if (!txn_identity_number(found.ptr, found.len, &answer.number, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&found, 1, 1); + std_vec_free_Checked(&keyed, 1, 1); + } + host_shared_Held_drop(&deltas); + if (!txn_memtable_branch_dropped(io, number, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&numbered, 1, 1); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return answer; +} + +/* txn_runs_tests_identity_is asserts at its line what identity_checkpointed answered: whether it held, and the pair's two keys found. */ +static void +txn_runs_tests_identity_is(uint32 line, txn_runs_tests_identity_t *answer, bool held) +{ + uint8 object[8]; + + std_u64_to_be_bytes(answer->object, object); + STD_ASSERT(line, 0, answer->held == held, "held"); + STD_ASSERT(line, 0, answer->found && answer->by_oid.len == 8 && memcmp(answer->by_oid.ptr, object, 8) == 0 && + answer->number == 16384, "(found, number) == (Some(object.to_be_bytes()), 16384)"); + if (answer->found) + std_vec_free_Checked(&answer->by_oid, 1, 1); +} + +/* + * PLAN item 10's runs: a database's identity pairs a transaction committed + * past the checkpoint's fold point recorded go to its checkpoint's identity + * run, and a start with its working state's delta gone loads them back, with + * their writer's outcome, so they read as they did. + */ +void +test_txn_runs_tests_a_checkpoints_identity_run_loads_its_pairs_back(void) +{ + SimcTestIo test_io = simc_io(); + txn_runs_tests_identity_t answer = txn_runs_tests_identity_checkpointed(&test_io.io, 9476, + UINT64CONST(1) << 61); + + txn_runs_tests_identity_is(__LINE__, &answer, true); + simc_TestIo_drop(&test_io); +} + +/* + * PLAN item 10's runs: a pair the checkpoint's save folds into the manifest + * is left out of its identity run, so a start loads no version of it back + * into the working state's delta and reads it from the manifest alone; a run + * that held it would load it again at every start of a database no later + * checkpoint saves. + */ +void +test_txn_runs_tests_a_pair_the_save_folds_is_left_out_of_the_identity_run(void) +{ + SimcTestIo test_io = simc_io(); + txn_runs_tests_identity_t answer = txn_runs_tests_identity_checkpointed(&test_io.io, 9480, 1); + + txn_runs_tests_identity_is(__LINE__, &answer, false); + simc_TestIo_drop(&test_io); +} + +/* txn_runs_tests_counted returns the count of a top-level transaction's status entry, 0 for none. */ +static uint64 +txn_runs_tests_counted(HostIoIo *io, HostXid top) +{ + TxnStatusEntry entry; + HostError error; + bool some; + + if (!txn_status_lookup(io, top, &some, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return some ? entry.count : 0; +} + +/* + * PRevPF 8's P1 on bugs row 164: identity pairs a start loads back from a + * checkpoint's run count at their writer's entry, as loaded rows and map + * versions do, so the next checkpoint's drop gives back exactly what the + * load took: the entry counts the pairs after the start and nothing after + * the drop. + */ +void +test_txn_runs_tests_a_starts_loaded_identity_pairs_count_at_their_writers_entry(void) +{ + static const uint8 zeros[0x200] = {0}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + TxnInsertPage page = {.bytes = 8160, .most = 291}; + HostIoWaits waits = host_io_Waits_testing(); + TxnMemtableVersionWriter writer = {0}; + TxnStatusOutcome committed = {.tag = TXN_STATUS_OUTCOME_COMMITTED}; + RepoRepo repo; + HostSharedReader reading; + HostXid sub; + HostXid top; + TxnStatusRoomed roomed; + TxnStatusEntry entry; + StdVec numbered; + ChunksErrorError failed; + HostError error; + HostLsn inserted; + uint64 object; + bool some; + bool settled; + bool removed; + + simc_end(false); + if (!repo_Repo_create(io, 9491, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + repo_Repo_drop(&repo); + if (!host_current_writer(HOST_IO_IO_IO_DOORS(io), &sub, &top, &error) || + !txn_status_roomed(io, top, &roomed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + writer.xid = sub.f0; + txn_identity_by_oid(TXN_IDENTITY_KIND_RELATION, 16384, &numbered); + { + TxnIdentityDatabase tree; + + repo = txn_runs_tests_reopened(io, 9491); + if (!host_shared_read(io, &reading, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + tree.f0 = &repo; + tree.f1 = &reading; + tree.f2 = 9491; + if (!txn_identity_derived(io, &waits, tree, writer, roomed, TXN_IDENTITY_KIND_RELATION, 5, + (const uint8 *) "t", 1, numbered.ptr, numbered.len, 16384, + txn_runs_tests_relation_keyed, NULL, NULL, &object, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + } + std_vec_free_Checked(&numbered, 1, 1); + simc_end(true); + if (!txn_written_ended(io, &error) || !txn_status_lookup(io, top, &some, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!some) + STD_OPTION_UNWRAP_FAILED(__LINE__, 0); + if (!host_fold_point(HOST_IO_IO_IO_DOORS(io), &committed.u.committed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + committed.u.committed += 0x100; + if (!txn_status_settled(io, top, entry.outcome, committed, &settled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, settled, "status::settled(io, top, entry.outcome, Committed(end)).unwrap()"); + STD_ASSERT_EQ_U64(__LINE__, 0, txn_runs_tests_counted(io, top), 2); + simc_checkpointed(UINT64CONST(0x100000a27)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000a28), &error) || !txn_memtable_branch_dropped(io, 9491, &error) || + !txn_status_remove(io, top, &removed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, removed, "status::remove(io, top).unwrap()"); + if (!txn_runs_loaded(io, &page, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ_U64(__LINE__, 0, txn_runs_tests_counted(io, top), 2); + if (!host_wal_insert(HOST_IO_IO_IO_DOORS(io), HOST_WAL_KIND_DATABASE_CHANGE, zeros, sizeof(zeros), &inserted, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_checkpointed(UINT64CONST(0x100000b27)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000b28), &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ_U64(__LINE__, 0, txn_runs_tests_counted(io, top), 0); + if (!txn_memtable_branch_dropped(io, 9491, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_TestIo_drop(&test_io); +} + +/* txn_runs_tests_deferred_closure_0 is the deferred case's pick of its storage's every spill file. */ +static bool +txn_runs_tests_deferred_closure_0(void *captures, const TxnSpillFile *file) +{ + (void) captures; + (void) file; + return true; +} + +/* + * Bugs row 228's sweep: a committed transaction's spill file, unlisted while + * a reader holds, is found unnamed and deferred, then listed again, as a + * fold's residual is listed after a checkpoint named its storage's files, so + * a later checkpoint names it and it leaves the deferred list; unlisted + * again once the reader ended, it is kept by the first checkpoint that finds + * it unnamed and by the second, whose epoch it then waits past, and removed + * by the next, never by the first sighting's old epoch. + */ +void +test_txn_runs_tests_a_deferred_spill_named_again_waits_for_two_new_sightings(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostStorage storage = {.tablespace = HOST_PG_DEFAULT, .database = 9752, .relation = 16384}; + TxnStatusOutcome committed = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = 0x0900}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostXid begun; + uint64 mine; + TxnMemtableOp op; + TxnMemtableVersionChange change = {.op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, + .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + TxnStatusRoomed roomed; + StdPathBuf dir; + StdVec listed; + StdVec gone; + HostLsn redo = UINT64CONST(0x100000a28); + HostError error; + bool spilled; + bool took; + static const struct + { + int checkpoints; + int unlist; + bool link; + bool drop; + size_t files; + const char *message; + } steps[] = {{1, 1, false, false, SIZE_MAX, NULL}, {2, 0, false, false, 2, "deferred while the reader holds"}, + {0, 0, true, false, SIZE_MAX, NULL}, {1, 0, false, true, SIZE_MAX, NULL}, + {2, 0, false, false, 2, "the first sighting since its naming keeps it"}, + {1, 0, false, false, 2, "the second sighting keeps it until its epoch passes"}, {1, 0, false, false, 0, "the spill is removed"}}; + + txn_runs_tests_repository(io, 9752, __LINE__, 9, __LINE__, 9, &repo, &reading); + begun = simc_begin(); + mine = begun.f0; + held = txn_runs_tests_stated(io, &repo, &reading, &storage, __LINE__, 9); + op = txn_memtable_testing_put((const uint8 *) "v", 1); + change.writer = (TxnMemtableVersionWriter) + { + .xid = mine + }; + if (!txn_memtable_table_push(io, host_shared_Held_record(&held), (const uint8 *) "k", 1, &change, &handle, &error) || + !txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + STD_ASSERT(__LINE__, 9, spilled, "crate::spill::spilled(io, &storage, crate::written::Of::Rows, mine).unwrap()"); + if (!txn_status_take(io, begun, committed, &roomed, &took, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_commit(begun); + txn_runs_spill_directory(9752, HOST_PG_DEFAULT, &dir); + for (size_t s = 0; s < lengthof(steps); s++) + { + for (int c = 0; c < steps[s].checkpoints; c++) + { + simc_checkpointed(redo - 1); + if (!txn_runs_checkpoint(io, redo, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + redo += 0x1000; + } + if (steps[s].unlist) + { + if (!txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, io, host_shared_Held_record(&held), &storage, + txn_runs_tests_deferred_closure_0, NULL, &listed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + } + if (steps[s].link) + { + txn_TupleU64KindU32OptionU32_t none = {0}; + + if (!txn_spill_linked(&TXN_SPILL_STATE_HOME_OPS, io, host_shared_Held_record(&held), &storage, + listed.ptr, listed.len, false, none, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&listed, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + } + if (steps[s].drop) + { + if (!txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, io, host_shared_Held_record(&held), &storage, + txn_runs_tests_deferred_closure_0, NULL, &gone, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + } + if (steps[s].message != NULL) + { + StdVec names; + bool some; + size_t files = 0; + + if (!host_list_dir_if_there(HOST_IO_IO_READ_DOORS(io), dir.bytes.ptr, dir.bytes.len, &some, &names, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; some && k < names.len; k++) + { + StdString *name = &STD_VEC_AT(&names, StdString, k); + + if (name->len < strlen(TXN_SPILL_MANIFEST) || + memcmp(name->ptr, TXN_SPILL_MANIFEST, strlen(TXN_SPILL_MANIFEST)) != 0) + files++; + std_String_drop(name); + } + if (some) + std_vec_free_Checked(&names, sizeof(StdString), _Alignof(StdString)); + if (files != steps[s].files) + std_panic(RUST_FILE, __LINE__, 9, steps[s].message, strlen(steps[s].message)); + } + } + std_vec_free_Checked(&dir.bytes, 1, 1); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + #endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/spill.c b/src/doltgres/txn/c/spill.c index 69a64f9e30f76..2bc3199638251 100644 --- a/src/doltgres/txn/c/spill.c +++ b/src/doltgres/txn/c/spill.c @@ -49,6 +49,7 @@ #include "doltgres/txn/memtable/table.h" #include "doltgres/txn/memtable/tuples.h" #include "doltgres/txn/memtable/version.h" +#include "doltgres/txn/record.h" #include "doltgres/txn/runs.h" #include "doltgres/txn/spill.h" #include "doltgres/txn/state.h" @@ -94,6 +95,13 @@ struct TxnSpillHomeOps size_t listed; size_t pended; size_t changed; + + /* + * taken is the atomic of the record's take generation: its take-outs of + * spilled versions from the memtable under way in its low half, and those + * ended in its high half. + */ + size_t taken; uint32 (*own) (HostSharedRecord record); bool (*described) (HostSharedRecord record, HostStorage *storage, TxnStatePersistence *persistence); bool (*kind) (HostIoIo *io, const HostSharedHeld *held, TxnWrittenOf of, TxnRunsKind *kind, @@ -101,20 +109,21 @@ struct TxnSpillHomeOps bool (*tree) (TxnWrittenOf of, const TxnMemtableTableTreeOps **result, HostError *error); bool (*untupled) (HostIoIo *io, HostSharedRecord record, const uint64 *tids, size_t tids_len, HostError *error); -}; -/* TXN_SPILL_IDENTITY is the error of a spill of a database's identity tree, which never spills. */ -#define TXN_SPILL_IDENTITY "a database's identity tree never spills" - -/* txn_spill_identity returns the error of a spill of a database's identity tree. */ -static HostError -txn_spill_identity(void) -{ - StdString message; + /* + * through returns the point the record's newest base is folded at, which + * a map's spill at or below it has passed: the fold that installed the + * base applied it or put its versions back. + */ + bool (*through) (HostIoIo *io, HostSharedRecord record, uint64 *result, HostError *error); - std_string_from_Checked(&message, TXN_SPILL_IDENTITY, strlen(TXN_SPILL_IDENTITY)); - return host_Error_fork((const uint8 *) "XX000", &message); -} + /* + * ended_number takes the number of a spill file of an ended transaction + * past past, as state's numbered does for a storage's state; a working + * state's chunk spills are only their running transactions'. + */ + uint32 (*ended_number) (HostSharedRecord record, uint32 past); +}; /* txn_spill_State_Home_described is State's described: the state's storage and persistence, or nothing for a state not filled. */ static bool @@ -160,17 +169,18 @@ txn_spill_State_Home_kind(HostIoIo *io, const HostSharedHeld *held, TxnWrittenOf case TXN_WRITTEN_OF_VALUES: *kind = TXN_RUNS_KIND_SPILLED_VALUES; return true; + case TXN_WRITTEN_OF_FREE: + *kind = TXN_RUNS_KIND_SPILLED_FREE; + return true; case TXN_WRITTEN_OF_IDENTITY: - *error = txn_spill_identity(); - return false; case TXN_WRITTEN_OF_CHUNKS: break; } - *error = txn_spill_internal("a storage's state has no chunk tree"); + *error = txn_spill_internal("a storage's state has no working state's tree"); return false; } -/* txn_spill_State_tree is the tree of a state's that copied, removed, and chain walk for an Of: its rows or its value map. */ +/* txn_spill_State_tree is the tree of a state's that copied, removed, and chain walk for an Of: its rows, its value map, or its free space map. */ static bool txn_spill_State_tree(TxnWrittenOf of, const TxnMemtableTableTreeOps **result, HostError *error) { @@ -182,13 +192,14 @@ txn_spill_State_tree(TxnWrittenOf of, const TxnMemtableTableTreeOps **result, Ho case TXN_WRITTEN_OF_VALUES: *result = &TXN_MEMTABLE_MAP_VALUES_TREE_OPS; return true; + case TXN_WRITTEN_OF_FREE: + *result = &TXN_MEMTABLE_MAP_FREE_TREE_OPS; + return true; case TXN_WRITTEN_OF_IDENTITY: - *error = txn_spill_identity(); - return false; case TXN_WRITTEN_OF_CHUNKS: break; } - *error = txn_spill_internal("a storage's state has no chunk tree"); + *error = txn_spill_internal("a storage's state has no working state's tree"); return false; } @@ -199,17 +210,27 @@ txn_spill_State_Home_untupled(HostIoIo *io, HostSharedRecord record, const uint6 return txn_memtable_tuples_dropped(io, record, tids, tids_len, error); } +/* txn_spill_State_Home_through is State's through: its newest base's read point. */ +static bool +txn_spill_State_Home_through(HostIoIo *io, HostSharedRecord record, uint64 *result, HostError *error) +{ + return txn_state_read_point(io, record, result, error); +} + const TxnSpillHomeOps TXN_SPILL_STATE_HOME_OPS = { .writing = TXN_STATE_SPILLS_WRITING_INDEX, .short_ = TXN_STATE_STATE_LOCK_INDEX, .listed = TXN_STATE_SPILLS_INDEX, .pended = TXN_STATE_SPILLS_PENDING_INDEX, .changed = TXN_STATE_SPILLS_CHANGED_INDEX, + .taken = TXN_STATE_SPILLS_TAKEN_INDEX, .own = txn_state_own, .described = txn_spill_State_Home_described, .kind = txn_spill_State_Home_kind, .tree = txn_spill_State_tree, .untupled = txn_spill_State_Home_untupled, + .through = txn_spill_State_Home_through, + .ended_number = txn_state_numbered, }; /* txn_spill_BranchDeltas_Home_described is BranchDeltas' described: a working state's chunk spills by its repository's chunk storage, logged. */ @@ -260,24 +281,98 @@ txn_spill_BranchDeltas_Home_untupled(HostIoIo *io, HostSharedRecord record, cons return true; } +/* txn_spill_BranchDeltas_Home_through is BranchDeltas' through: 0, since a working state's chunk spills hold no map's versions. */ +static bool +txn_spill_BranchDeltas_Home_through(HostIoIo *io, HostSharedRecord record, uint64 *result, HostError *error) +{ + *result = 0; + return true; +} + +/* txn_spill_BranchDeltas_Home_ended_number is BranchDeltas' ended_number: past itself. */ +static uint32 +txn_spill_BranchDeltas_Home_ended_number(HostSharedRecord record, uint32 past) +{ + return past; +} + const TxnSpillHomeOps TXN_SPILL_BRANCH_DELTAS_HOME_OPS = { .writing = TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_WRITING_INDEX, .short_ = TXN_MEMTABLE_BRANCH_DELTAS_LOCK_INDEX, .listed = TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_INDEX, .pended = TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_PENDING_INDEX, .changed = TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_CHANGED_INDEX, + .taken = TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_TAKEN_INDEX, .own = txn_memtable_branch_own, .described = txn_spill_BranchDeltas_Home_described, .kind = txn_spill_BranchDeltas_Home_kind, .tree = txn_spill_BranchDeltas_tree, .untupled = txn_spill_BranchDeltas_Home_untupled, + .through = txn_spill_BranchDeltas_Home_through, + .ended_number = txn_spill_BranchDeltas_Home_ended_number, +}; + +/* txn_spill_Identity_Home_described is the identity spills' described: a working state's identity spills by its database's identity storage, logged. */ +static bool +txn_spill_Identity_Home_described(HostSharedRecord record, HostStorage *storage, TxnStatePersistence *persistence) +{ + uint32 repository; + + if (!txn_memtable_branch_repository(record, &repository)) + return false; + *storage = txn_spill_identity_storage(repository); + *persistence = TXN_STATE_PERSISTENCE_PERMANENT; + return true; +} + +/* txn_spill_Identity_tree is the tree the identity spills' home spills: a working state's identity tree alone. */ +static bool +txn_spill_Identity_tree(TxnWrittenOf of, const TxnMemtableTableTreeOps **result, HostError *error) +{ + if (of == TXN_WRITTEN_OF_IDENTITY) + { + *result = &TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS; + return true; + } + *error = txn_spill_internal("a working state's identity spills hold only its identity tree"); + return false; +} + +/* txn_spill_Identity_Home_kind is the identity spills' kind: an identity spill, whose header holds no tail. */ +static bool +txn_spill_Identity_Home_kind(HostIoIo *io, const HostSharedHeld *held, TxnWrittenOf of, TxnRunsKind *kind, + TxnRunsTail *tail, HostError *error) +{ + const TxnMemtableTableTreeOps *tree; + + if (!txn_spill_Identity_tree(of, &tree, error)) + return false; + *kind = TXN_RUNS_KIND_SPILLED_IDENTITY; + tail->tag = TXN_RUNS_TAIL_NONE; + return true; +} + +const TxnSpillHomeOps TXN_SPILL_IDENTITY_HOME_OPS = { + .writing = TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_WRITING_INDEX, + .short_ = TXN_MEMTABLE_BRANCH_DELTAS_LOCK_INDEX, + .listed = TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_INDEX, + .pended = TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_PENDING_INDEX, + .changed = TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_CHANGED_INDEX, + .taken = TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_TAKEN_INDEX, + .own = txn_memtable_branch_own, + .described = txn_spill_Identity_Home_described, + .kind = txn_spill_Identity_Home_kind, + .tree = txn_spill_Identity_tree, + .untupled = txn_spill_BranchDeltas_Home_untupled, + .through = txn_spill_BranchDeltas_Home_through, + .ended_number = txn_spill_BranchDeltas_Home_ended_number, }; /* txn_spill_Home_copied hands every key of one of a home's trees that has a version to a walk's step, with no killed version. */ static bool txn_spill_Home_copied(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, TxnWrittenOf of, bool (*each) (void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *records, - size_t records_len, const txn_TupleHandleU64_t *handles, size_t handles_len, + size_t records_len, const txn_TupleHandleU64Bool_t *handles, size_t handles_len, HostError *error), void *captures, HostError *error) { @@ -333,14 +428,34 @@ txn_spill_chunks_repository(const HostStorage *storage, uint32 *result) return true; } +/* txn_spill_identity_storage returns the storage a database's identity tree is named by: its database with no tablespace and relation 1, which no storage has (bugs row 291). */ +HostStorage +txn_spill_identity_storage(uint32 repository) +{ + HostStorage storage = {.database = repository, .relation = 1}; + + return storage; +} + +/* txn_spill_identity_repository returns the repository whose identity spills a storage names, or nothing for another storage. */ +static bool +txn_spill_identity_repository(const HostStorage *storage, uint32 *result) +{ + if (storage->tablespace != 0 || storage->relation != 1 || storage->database == 0) + return false; + *result = storage->database; + return true; +} + /* TxnSpillFoundTag is Found's variants. */ typedef enum TxnSpillFoundTag { TXN_SPILL_FOUND_STATE, TXN_SPILL_FOUND_CHUNKS, + TXN_SPILL_FOUND_IDENTITY, } TxnSpillFoundTag; -/* TxnSpillFound is Found: the home of a storage's spills, its state, or its repository's working state's delta. */ +/* TxnSpillFound is Found: the home of a storage's spills, its state, or its repository's working state's delta for its chunk or identity spills. */ typedef struct TxnSpillFound { TxnSpillFoundTag tag; @@ -351,6 +466,8 @@ typedef struct TxnSpillFound static const TxnSpillHomeOps * txn_spill_Found_home(const TxnSpillFound *found) { + if (found->tag == TXN_SPILL_FOUND_IDENTITY) + return &TXN_SPILL_IDENTITY_HOME_OPS; return found->tag == TXN_SPILL_FOUND_STATE ? &TXN_SPILL_STATE_HOME_OPS : &TXN_SPILL_BRANCH_DELTAS_HOME_OPS; } @@ -365,6 +482,11 @@ txn_spill_found(HostIoIo *io, const HostStorage *storage, bool *some, TxnSpillFo result->tag = TXN_SPILL_FOUND_CHUNKS; return txn_memtable_branch_find(HOST_SHARED_IO_WITHIN(io), repository, some, &result->held, error); } + if (txn_spill_identity_repository(storage, &repository)) + { + result->tag = TXN_SPILL_FOUND_IDENTITY; + return txn_memtable_branch_find(HOST_SHARED_IO_WITHIN(io), repository, some, &result->held, error); + } result->tag = TXN_SPILL_FOUND_STATE; return txn_state_found(io, storage, some, &result->held, error); } @@ -532,17 +654,18 @@ static bool txn_spill_File_PartialEq_eq(const TxnSpillFile *self, const TxnSpillFile *other) { return self->xid == other->xid && self->kind == other->kind && self->number == other->number && - self->len == other->len && self->hash == other->hash && self->bits == other->bits; + self->len == other->len && self->hash == other->hash && self->bits == other->bits && + self->passed == other->passed && self->from == other->from; } -/* txn_spill_File_line appends a file's line in its storage's manifest to a buffer: its xid, kind, number, length, xxh3, and bits. */ +/* txn_spill_File_line appends a file's line in its storage's manifest to a buffer: its xid, kind, number, length, xxh3, bits, abort's end, and residual's point. */ static void txn_spill_File_line(const TxnSpillFile *self, StdString *lines) { - char line[128]; + char line[192]; int len = snprintf(line, sizeof(line), UINT64_FORMAT " %u %u " UINT64_FORMAT " " UINT64_FORMAT " " - UINT64_FORMAT "\n", self->xid, (unsigned) self->kind, self->number, self->len, - self->hash, self->bits); + UINT64_FORMAT " " UINT64_FORMAT " " UINT64_FORMAT "\n", self->xid, (unsigned) self->kind, + self->number, self->len, self->hash, self->bits, self->passed, self->from); std_vec_extend_from_slice_Checked(lines, line, (size_t) len, 1, 1); } @@ -551,11 +674,11 @@ txn_spill_File_line(const TxnSpillFile *self, StdString *lines) static bool txn_spill_File_parsed(const char *line, size_t len, TxnSpillFile *result) { - uint64 words[6]; + uint64 words[8]; size_t at = 0; StdParseIntError failed; - for (size_t count = 0; count < 6; count++) + for (size_t count = 0; count < lengthof(words); count++) { size_t end = at; @@ -576,6 +699,8 @@ txn_spill_File_parsed(const char *line, size_t len, TxnSpillFile *result) result->len = words[3]; result->hash = words[4]; result->bits = words[5]; + result->passed = words[6]; + result->from = words[7]; return result->bits > 0 && result->bits % 64 == 0; } @@ -761,25 +886,16 @@ txn_spill_read(HostIoReadDoors io, HostSharedPauses pauses, const StdPathBuf *pa return frame.ok; } -/* txn_spill_changed_frame_0_t is changed's state across its pad: the manifest's path, its writers' lock, its files, and their write aside, while live. */ -typedef struct txn_spill_changed_frame_0_t +/* txn_spill_rewrote_frame_0_t is rewrote's state across its pad: the manifest's lines, their write aside, and the record's lock, while live. */ +typedef struct txn_spill_rewrote_frame_0_t { const TxnSpillHomeOps *home; - HostIoIo *io; + HostSharedLongIo *io; HostSharedRecord record; - const HostStorage *storage; - void (*edit) (void *captures, StdVec *files); - void *captures; + const StdPathBuf *path; + const StdVec *files; HostError *error; bool ok; - StdPathBuf path; - bool path_live; - HostSharedLongGuard writing; - bool writing_live; - HostSharedLongIo long_io; - /* files is a Vec. */ - StdVec files; - bool files_live; StdString lines; bool lines_live; StdPathBuf aside; @@ -788,13 +904,13 @@ typedef struct txn_spill_changed_frame_0_t bool written_live; HostSharedGuard exclusive; bool exclusive_live; -} txn_spill_changed_frame_0_t; +} txn_spill_rewrote_frame_0_t; -/* txn_spill_changed_pad_0 is changed's pad: what it holds, while live. */ +/* txn_spill_rewrote_pad_0 is rewrote's pad: what it holds, while live. */ static void -txn_spill_changed_pad_0(void *frame_) +txn_spill_rewrote_pad_0(void *frame_) { - txn_spill_changed_frame_0_t *frame = frame_; + txn_spill_rewrote_frame_0_t *frame = frame_; if (frame->exclusive_live) { @@ -816,36 +932,21 @@ txn_spill_changed_pad_0(void *frame_) frame->lines_live = false; std_vec_free_Checked(&frame->lines, 1, 1); } - if (frame->files_live) - { - frame->files_live = false; - std_vec_free_Checked(&frame->files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - } - if (frame->writing_live) - { - frame->writing_live = false; - host_shared_LongGuard_drop(&frame->writing); - } - if (frame->path_live) - { - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - } } -/* txn_spill_changed_written writes the frame's files' lines whole beside the manifest and renames them over it. */ +/* txn_spill_rewrote_written writes the frame's files' lines whole beside the manifest and renames them over it. */ static bool -txn_spill_changed_written(txn_spill_changed_frame_0_t *frame) +txn_spill_rewrote_written(txn_spill_rewrote_frame_0_t *frame) { - HostIoIoDoors io = HOST_SHARED_LONG_IO_IO_DOORS(&frame->long_io); + HostIoIoDoors io = HOST_SHARED_LONG_IO_IO_DOORS(frame->io); ChunksErrorError failed; bool some; std_vec_new(&frame->lines, 1, 1); frame->lines_live = true; - for (size_t k = 0; k < frame->files.len; k++) - txn_spill_File_line(&STD_VEC_AT(&frame->files, TxnSpillFile, k), &frame->lines); - std_path_with_extension(frame->path.bytes.ptr, frame->path.bytes.len, "new", 3, &frame->aside); + for (size_t k = 0; k < frame->files->len; k++) + txn_spill_File_line(&STD_VEC_AT(frame->files, TxnSpillFile, k), &frame->lines); + std_path_with_extension(frame->path->bytes.ptr, frame->path->bytes.len, "new", 3, &frame->aside); frame->aside_live = true; if (!host_create_if_absent(io, frame->aside.bytes.ptr, frame->aside.bytes.len, &some, &frame->written, frame->error)) @@ -868,8 +969,8 @@ txn_spill_changed_written(txn_spill_changed_frame_0_t *frame) } frame->written_live = false; host_File_drop(&frame->written); - if (!chunks_file_rename(io, frame->aside.bytes.ptr, frame->aside.bytes.len, frame->path.bytes.ptr, - frame->path.bytes.len, &failed)) + if (!chunks_file_rename(io, frame->aside.bytes.ptr, frame->aside.bytes.len, frame->path->bytes.ptr, + frame->path->bytes.len, &failed)) { *frame->error = txn_base_storage(&failed); return false; @@ -877,15 +978,118 @@ txn_spill_changed_written(txn_spill_changed_frame_0_t *frame) return true; } +/* txn_spill_rewrote_body_0 is rewrote's body. */ +static void +txn_spill_rewrote_body_0(void *frame_) +{ + txn_spill_rewrote_frame_0_t *frame = frame_; + uint64 generation; + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + HostSharedOp exchange = {.tag = HOST_SHARED_OP_EXCHANGE}; + + if (frame->files->len == 0) + { + ChunksErrorError failed; + bool removed; + + if (!chunks_file_remove_if_there(HOST_SHARED_LONG_IO_IO_DOORS(frame->io), frame->path->bytes.ptr, + frame->path->bytes.len, &removed, &failed)) + { + *frame->error = txn_base_storage(&failed); + txn_spill_rewrote_pad_0(frame); + return; + } + } + else if (!txn_spill_rewrote_written(frame)) + { + txn_spill_rewrote_pad_0(frame); + return; + } + if (!txn_state_spills_generation(&generation, frame->error) || + !host_shared_LongIo_lock(frame->io, frame->home->short_, &frame->exclusive, frame->error)) + { + txn_spill_rewrote_pad_0(frame); + return; + } + frame->exclusive_live = true; + exchange.u.exchange.f0 = host_shared_Record_atomic(frame->record, frame->home->listed, load); + exchange.u.exchange.f1 = (uint64) frame->files->len; + host_shared_Record_atomic(frame->record, frame->home->listed, exchange); + exchange.u.exchange.f0 = host_shared_Record_atomic(frame->record, frame->home->changed, load); + exchange.u.exchange.f1 = generation; + host_shared_Record_atomic(frame->record, frame->home->changed, exchange); + frame->ok = true; + txn_spill_rewrote_pad_0(frame); +} + +/* + * txn_spill_rewrote writes a storage's manifest file under its writers' lock, + * its files' lines through a file set aside and renamed over it, unsynced, or + * removes it when it lists none, and then, under the record's lock held + * exclusively for the in-memory steps alone, moves the count and the + * generation its record holds. + */ +static bool +txn_spill_rewrote(const TxnSpillHomeOps *home, HostSharedLongIo *io, HostSharedRecord record, const StdPathBuf *path, + const StdVec *files, HostError *error) +{ + txn_spill_rewrote_frame_0_t frame = {.home = home, .io = io, .record = record, .path = path, .files = files, + .error = error}; + + std_landing(&frame, txn_spill_rewrote_body_0, txn_spill_rewrote_pad_0); + return frame.ok; +} + +/* txn_spill_changed_frame_0_t is changed's state across its pad: the manifest's path, its writers' lock, and its files, while live. */ +typedef struct txn_spill_changed_frame_0_t +{ + const TxnSpillHomeOps *home; + HostIoIo *io; + HostSharedRecord record; + const HostStorage *storage; + bool (*edit) (void *captures, HostSharedLongIo *io, StdVec *files, HostError *error); + void *captures; + HostError *error; + bool ok; + StdPathBuf path; + bool path_live; + HostSharedLongGuard writing; + bool writing_live; + HostSharedLongIo long_io; + /* files is a Vec. */ + StdVec files; + bool files_live; +} txn_spill_changed_frame_0_t; + +/* txn_spill_changed_pad_0 is changed's pad: what it holds, while live. */ +static void +txn_spill_changed_pad_0(void *frame_) +{ + txn_spill_changed_frame_0_t *frame = frame_; + + if (frame->files_live) + { + frame->files_live = false; + std_vec_free_Checked(&frame->files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + } + if (frame->writing_live) + { + frame->writing_live = false; + host_shared_LongGuard_drop(&frame->writing); + } + if (frame->path_live) + { + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + } +} + /* txn_spill_changed_body_0 is changed's body. */ static void txn_spill_changed_body_0(void *frame_) { txn_spill_changed_frame_0_t *frame = frame_; HostIoIoDoors io; - uint64 generation; - HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; - HostSharedOp exchange = {.tag = HOST_SHARED_OP_EXCHANGE}; txn_spill_manifest(frame->home, frame->record, frame->storage, &frame->path); frame->path_live = true; @@ -905,57 +1109,28 @@ txn_spill_changed_body_0(void *frame_) return; } frame->files_live = true; - if (!host_pause(HOST_SHARED_LONG_IO_PAUSES(&frame->long_io), "doltgres-spill-manifest-read", frame->error)) - { - txn_spill_changed_pad_0(frame); - return; - } - frame->edit(frame->captures, &frame->files); - if (frame->files.len == 0) - { - ChunksErrorError failed; - bool removed; - - if (!chunks_file_remove_if_there(io, frame->path.bytes.ptr, frame->path.bytes.len, &removed, &failed)) - { - *frame->error = txn_base_storage(&failed); - txn_spill_changed_pad_0(frame); - return; - } - } - else if (!txn_spill_changed_written(frame)) - { - txn_spill_changed_pad_0(frame); - return; - } - if (!txn_state_spills_generation(&generation, frame->error) || - !host_shared_LongIo_lock(&frame->long_io, frame->home->short_, &frame->exclusive, frame->error)) + if (!host_pause(HOST_SHARED_LONG_IO_PAUSES(&frame->long_io), "doltgres-spill-manifest-read", frame->error) || + !frame->edit(frame->captures, &frame->long_io, &frame->files, frame->error) || + !txn_spill_rewrote(frame->home, &frame->long_io, frame->record, &frame->path, &frame->files, frame->error)) { txn_spill_changed_pad_0(frame); return; } - frame->exclusive_live = true; - exchange.u.exchange.f0 = host_shared_Record_atomic(frame->record, frame->home->listed, load); - exchange.u.exchange.f1 = (uint64) frame->files.len; - host_shared_Record_atomic(frame->record, frame->home->listed, exchange); - exchange.u.exchange.f0 = host_shared_Record_atomic(frame->record, frame->home->changed, load); - exchange.u.exchange.f1 = generation; - host_shared_Record_atomic(frame->record, frame->home->changed, exchange); frame->ok = true; txn_spill_changed_pad_0(frame); } /* * txn_spill_changed changes a storage's spills' manifest under its writers' - * lock: it reads the manifest file, edits its list, writes the list whole - * beside it and renames it over the manifest, unsynced, or removes the file - * once the list is empty; then, under the record's lock held exclusively for - * the in-memory steps alone, it stores the list's count and a new - * generation. What the edit answers it leaves in its captures. + * lock: it reads the manifest file, edits its list with an edit that may + * write files under the lock and fail, which leaves the manifest as it was, + * and rewrites the manifest with the edited list (rewrote). What the edit + * answers it leaves in its captures. */ static bool txn_spill_changed(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, const HostStorage *storage, - void (*edit) (void *captures, StdVec *files), void *captures, HostError *error) + bool (*edit) (void *captures, HostSharedLongIo *io, StdVec *files, HostError *error), + void *captures, HostError *error) { txn_spill_changed_frame_0_t frame = {.home = home, .io = io, .record = record, .storage = storage, .edit = edit, .captures = captures, .error = error}; @@ -973,12 +1148,17 @@ typedef struct txn_spill_linked_captures_t txn_TupleU64KindU32OptionU32_t replacing; } txn_spill_linked_captures_t; -/* txn_spill_linked_closure_0 is linked's edit: the files the merge replaces out, then each linked file once. */ -static void -txn_spill_linked_closure_0(void *captures, StdVec *files) +/* + * txn_spill_linked_closure_0 is linked's edit: the files the merge replaces + * out, then each linked file once, at the place of the newest file the + * merge replaces, or last when it replaces none. + */ +static bool +txn_spill_linked_closure_0(void *captures, HostSharedLongIo *io, StdVec *files, HostError *error) { txn_spill_linked_captures_t *c = captures; size_t kept = 0; + size_t at = files->len; for (size_t k = 0; k < files->len; k++) { @@ -986,6 +1166,8 @@ txn_spill_linked_closure_0(void *captures, StdVec *files) if (!c->has_replacing || !txn_spill_replaced_by(&c->replacing, 1, file)) STD_VEC_AT(files, TxnSpillFile, kept++) = *file; + else + at = kept; } files->len = kept; for (size_t j = 0; j < c->linking_len; j++) @@ -995,14 +1177,21 @@ txn_spill_linked_closure_0(void *captures, StdVec *files) for (size_t k = 0; k < files->len && !listed; k++) listed = txn_spill_File_PartialEq_eq(&STD_VEC_AT(files, TxnSpillFile, k), &c->linking[j]); if (!listed) - std_vec_push_Checked(files, &c->linking[j], sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + { + std_vec_insert(files, at, &c->linking[j], sizeof(TxnSpillFile), _Alignof(TxnSpillFile), RUST_FILE, 523, + 23); + at++; + } } + return true; } /* * txn_spill_linked lists a spill's files in its storage's manifest, the * newest, a row spill's tuple-id file after its row file, each once; a - * merge's files take the place of the files it replaces, in the same change. + * merge's files take the place of the files it replaces, in the same change, + * at the newest one's place, so its versions stay below every later file's + * of their keys (bugs row 228). */ bool txn_spill_linked(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, const HostStorage *storage, @@ -1023,8 +1212,8 @@ typedef struct txn_spill_removed_captures_t } txn_spill_removed_captures_t; /* txn_spill_removed_closure_0 is removed's edit: the files its test picks out, kept in order, and the rest left. */ -static void -txn_spill_removed_closure_0(void *captures, StdVec *files) +static bool +txn_spill_removed_closure_0(void *captures, HostSharedLongIo *io, StdVec *files, HostError *error) { txn_spill_removed_captures_t *c = captures; size_t kept = 0; @@ -1039,6 +1228,7 @@ txn_spill_removed_closure_0(void *captures, StdVec *files) STD_VEC_AT(files, TxnSpillFile, kept++) = file; } files->len = kept; + return true; } /* txn_spill_removed_frame_0_t is removed's state across its pad: the files it takes, while live. */ @@ -1086,7 +1276,7 @@ txn_spill_removed_body_0(void *frame_) } /* txn_spill_removed takes the files of a storage's spills that a test picks out of its manifest, answering them. */ -static bool +bool txn_spill_removed(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, const HostStorage *storage, bool (*picked) (void *captures, const TxnSpillFile *file), void *picked_captures, StdVec *result, HostError *error) @@ -1542,7 +1732,7 @@ txn_spill_hits(HostIoIo *io, const StdPathBuf *path, const TxnSpillFile *file, c } /* TXN_SPILL_PENDING_PROC_SLOTS_OPS is Pending's ProcSlots: each process's slot naming its one spill no manifest lists yet. */ -const HostSharedProcSlotsOps TXN_SPILL_PENDING_PROC_SLOTS_OPS = {.id = UINT64CONST(0x74786e000015), .words = 8}; +const HostSharedProcSlotsOps TXN_SPILL_PENDING_PROC_SLOTS_OPS = {.id = UINT64CONST(0x74786e000015), .words = 9}; /* The slot's words: its storage's database and relation, tablespace, xid, numbered word, sources, unspilled bytes, turn, and top. */ #define TXN_SPILL_PENDING_STORAGE 0 @@ -1554,6 +1744,9 @@ const HostSharedProcSlotsOps TXN_SPILL_PENDING_PROC_SLOTS_OPS = {.id = UINT64CON #define TXN_SPILL_PENDING_TURN 6 #define TXN_SPILL_PENDING_TOP 7 +/* TXN_SPILL_PENDING_TAKING is the slot's word naming, one past its place, the record whose take generation counts the process's take-out under way. */ +#define TXN_SPILL_PENDING_TAKING 8 + /* txn_spill_mine returns the backend's own slot of Pending. */ static bool txn_spill_mine(HostIoIo *io, HostSharedMine *result, HostError *error) @@ -1738,6 +1931,35 @@ txn_spill_pending(HostIoIo *io, bool *some, TxnSpillPended *result, HostError *e return true; } +/* TXN_SPILL_TAKE_ENDED is what an ended take-out adds to its record's take generation: one to the high half. */ +#define TXN_SPILL_TAKE_ENDED ((uint64) 1 << 32) + +/* + * txn_spill_take_ended ends the take-out under way the process's slot names, + * if any, in its record's take generation, one fewer under way and one more + * ended, then clears the slot's word naming it; the one place that clears + * the word, so every way a slot is cleared ends its take-out. A record gone + * took its generation with it. + */ +static bool +txn_spill_take_ended(const TxnSpillHomeOps *home, HostIoIo *io, const HostSharedRecord *record, HostError *error) +{ + HostSharedMine mine; + HostSharedOp add = {.tag = HOST_SHARED_OP_ADD, .u.add = TXN_SPILL_TAKE_ENDED - 1}; + uint64 taking; + + if (!txn_spill_mine(io, &mine, error)) + return false; + taking = host_shared_Mine_load(&mine, TXN_SPILL_PENDING_TAKING); + if (taking == 0) + return true; + Assert(record == NULL || taking == record->place + 1); + if (record != NULL) + host_shared_Record_atomic(*record, home->taken, add); + host_shared_Mine_store(&mine, TXN_SPILL_PENDING_TAKING, 0); + return true; +} + /* txn_spill_settled clears the process's slot once its spill is listed or its transaction ended, then subtracts from the record's count of pending spills. */ static bool txn_spill_settled(const TxnSpillHomeOps *home, HostIoIo *io, const HostSharedRecord *state, HostError *error) @@ -1745,7 +1967,7 @@ txn_spill_settled(const TxnSpillHomeOps *home, HostIoIo *io, const HostSharedRec HostSharedMine mine; HostSharedOp sub = {.tag = HOST_SHARED_OP_ADD, .u.add = PG_UINT64_MAX}; - if (!txn_spill_mine(io, &mine, error)) + if (!txn_spill_take_ended(home, io, state, error) || !txn_spill_mine(io, &mine, error)) return false; host_shared_Mine_store(&mine, TXN_SPILL_PENDING_NUMBERED, 0); host_shared_Mine_store(&mine, TXN_SPILL_PENDING_SOURCES, 0); @@ -1754,12 +1976,96 @@ txn_spill_settled(const TxnSpillHomeOps *home, HostIoIo *io, const HostSharedRec return true; } -/* txn_spill_pended_frame_0_t is pended's state across its pad: the processes and the spills found, while live. */ -typedef struct txn_spill_pended_frame_0_t +/* txn_spill_Taking_Exit_run is Taking's Exit: a take-out an exit stopped settled, as an abort settles it. */ +static bool +txn_spill_Taking_Exit_run(HostIoIo *io, HostError *error) { - const TxnSpillHomeOps *home; - HostSharedRecord state; - const HostStorage *storage; + return txn_spill_take_settled(io, error); +} + +/* TXN_SPILL_TAKING_TYPE_ID is Taking's identity, by which its exit work is registered once. */ +static const StdTypeId TXN_SPILL_TAKING_TYPE_ID = {"txn::spill::Taking", 0, 1, NULL}; + +/* TXN_SPILL_TAKING_EXIT_OPS is Taking's Exit. */ +static const HostExitOps TXN_SPILL_TAKING_EXIT_OPS = {txn_spill_Taking_Exit_run}; + +/* + * txn_spill_take_began counts a take-out of a spill's versions from the + * memtable under way in its record's take generation and names the record in + * the process's slot, before the spill is pended, so that readers wait for + * its end, with the exit work that settles it listed first. + */ +static bool +txn_spill_take_began(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, HostError *error) +{ + HostSharedMine mine; + HostSharedOp add = {.tag = HOST_SHARED_OP_ADD, .u.add = 1}; + + if (!host_at_exit(&TXN_SPILL_TAKING_TYPE_ID, &TXN_SPILL_TAKING_EXIT_OPS, error) || !txn_spill_mine(io, &mine, error)) + return false; + if (host_shared_Mine_load(&mine, TXN_SPILL_PENDING_TAKING) != 0) + { + *error = txn_spill_internal("a process began a take-out while another was under way"); + return false; + } + host_shared_Mine_store(&mine, TXN_SPILL_PENDING_TAKING, STD_ADD(u64, record.place, 1, 0, 0)); + host_shared_Record_atomic(record, home->taken, add); + return true; +} + +/* TXN_SPILL_TAKE_NAP_MICROS is how long a reader naps on its latch before it looks again whether a take-out ended. */ +#define TXN_SPILL_TAKE_NAP_MICROS 1000 + +/* + * txn_spill_taken runs a read of a key's versions in a home's memtable and + * then in its spills again until no take-out of the home began or ended + * between the read's start and its spills' listing, whose take generation + * the read stores, dropping what the read gave before each rerun, so that a + * read finds each version a spill moves once. While another process's + * take-out is under way it waits on that take-out's end, napping; the + * process's own reads it at once. + */ +bool +txn_spill_taken(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, + bool (*read) (void *captures, uint64 *listed, HostError *error), void (*dropped) (void *captures), + void *captures, HostError *error) +{ + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + + for (;;) + { + uint64 before = host_shared_Record_atomic(record, home->taken, load); + uint64 listed = before; + + if ((uint32) before != 0) + { + HostSharedMine mine; + uint64 own; + + if (!txn_spill_mine(io, &mine, error)) + return false; + own = host_shared_Mine_load(&mine, TXN_SPILL_PENDING_TAKING) == STD_ADD(u64, record.place, 1, 0, 0); + if ((uint32) before > own) + { + if (!host_nap(HOST_IO_IO_IO_DOORS(io), std_Duration_from_micros(TXN_SPILL_TAKE_NAP_MICROS), error)) + return false; + continue; + } + } + if (!read(captures, &listed, error)) + return false; + if (listed == before) + return true; + dropped(captures); + } +} + +/* txn_spill_pended_frame_0_t is pended's state across its pad: the processes and the spills found, while live. */ +typedef struct txn_spill_pended_frame_0_t +{ + const TxnSpillHomeOps *home; + HostSharedRecord state; + const HostStorage *storage; StdVec *result; HostError *error; bool ok; @@ -2034,6 +2340,296 @@ txn_spill_replayed_locked(HostIoIo *io, void (*step) (void *captures, StdVec *re std_landing(&frame, txn_spill_replayed_body_0, txn_spill_replayed_pad_0); } +/* taken at Vec and IoMutex>, which spill.c alone uses. */ +#define HOST_LOCK_SCOPE static +#define HOST_LOCK_DECLARE +#define HOST_LOCK_DEFINE +#define HOST_LOCK_TAKEN_TS VecU64 +#define HOST_LOCK_TAKEN_T StdVec +#define HOST_LOCK_TAKEN_DEFAULT(place) std_vec_new((place), sizeof(uint64), _Alignof(uint64)) +#define HOST_LOCK_TAKEN_DROP(place) std_vec_free_Checked((place), sizeof(uint64), _Alignof(uint64)) +#define HOST_LOCK_TAKEN_TYPE TXN_SPILL_SPILLERS_LOCK_TYPE +#define HOST_LOCK_IO_MUTEX +#define HOST_LOCK_TS VecU64 +#define HOST_LOCK_T StdVec +#define HOST_LOCK_DEFAULT(place) std_vec_new((place), sizeof(uint64), _Alignof(uint64)) +#include "doltgres/host/lock_template.h" + +/* txn_spill_IoMutexVecU64_drop is IoMutex>' drop glue: its value's. */ +static void +txn_spill_IoMutexVecU64_drop(void *value) +{ + HostLockIoMutexVecU64 *self = value; + + std_vec_free_Checked(&self->value, sizeof(uint64), _Alignof(uint64)); +} + +/* TXN_SPILL_SPILLERS_TYPE is IoMutex>' size, Default, and drop. */ +static const StdDefaultType TXN_SPILL_SPILLERS_TYPE = {sizeof(HostLockIoMutexVecU64), + _Alignof(HostLockIoMutexVecU64), host_lock_IoMutexVecU64_Default_default, +txn_spill_IoMutexVecU64_drop}; + +/* + * TXN_SPILL_FREE_SPILLERS is FREE_SPILLERS: the startup process's xids whose + * free space map spills it listed, by its load of a checkpoint's runs or a + * replayed spill, so that a replayed abort sweeps the states for its + * transaction's spills only when it has some; its BackendLocal header and + * the process's value, which follows it. + */ +static struct +{ + HostLocalBackendLocal local; + HostLockIoMutexVecU64 process; +} TXN_SPILL_FREE_SPILLERS = {HOST_LOCAL_BACKEND_LOCAL_NEW}; + +/* txn_spill_spillers_frame_0_t is the state of a step under FREE_SPILLERS' lock across its pad: the local and its guard, while live. */ +typedef struct txn_spill_spillers_frame_0_t +{ + HostIoIo *io; + void (*step) (void *captures, StdVec *spillers); + void *captures; + HostLocalLocal local; + bool local_live; + HostLockLockedRwLockWriteGuard guard; + bool guard_live; +} txn_spill_spillers_frame_0_t; + +/* txn_spill_spillers_pad_0 is the pad of a step under FREE_SPILLERS' lock: the guard, then the local, while live. */ +static void +txn_spill_spillers_pad_0(void *frame_) +{ + txn_spill_spillers_frame_0_t *frame = frame_; + + if (frame->guard_live) + { + frame->guard_live = false; + host_lock_LockedRwLockWriteGuard_drop(&frame->guard); + } + if (frame->local_live) + { + frame->local_live = false; + host_local_Local_drop(&frame->local, &TXN_SPILL_SPILLERS_TYPE); + } +} + +/* txn_spill_spillers_body_0 runs a step on FREE_SPILLERS under its lock. */ +static void +txn_spill_spillers_body_0(void *frame_) +{ + txn_spill_spillers_frame_0_t *frame = frame_; + + frame->local = host_local_BackendLocal_get(&TXN_SPILL_FREE_SPILLERS.local, &TXN_SPILL_SPILLERS_TYPE); + frame->local_live = true; + frame->guard = host_lock_IoMutex_lock((HostLockIoMutex *) host_local_Local_Deref_deref(&frame->local, + &TXN_SPILL_SPILLERS_TYPE), + &TXN_SPILL_SPILLERS_LOCK_TYPE, &frame->io->held); + frame->guard_live = true; + frame->step(frame->captures, host_lock_LockedRwLockWriteGuard_DerefMut_deref_mut(&frame->guard)); + txn_spill_spillers_pad_0(frame); +} + +/* txn_spill_spillers_locked runs a step on the process's FREE_SPILLERS under its lock. */ +static void +txn_spill_spillers_locked(HostIoIo *io, void (*step) (void *captures, StdVec *spillers), void *captures) +{ + txn_spill_spillers_frame_0_t frame = {.io = io, .step = step, .captures = captures}; + + std_landing(&frame, txn_spill_spillers_body_0, txn_spill_spillers_pad_0); +} + +/* txn_spill_spillers_t is the steps' captures on FREE_SPILLERS: the xid, and whether the step found or recorded it. */ +typedef struct txn_spill_spillers_t +{ + uint64 xid; + bool done; +} txn_spill_spillers_t; + +/* txn_spill_spilled_free_closure_0 is spilled_free's step: the xid recorded once, done unless the list's growth failed. */ +static void +txn_spill_spilled_free_closure_0(void *captures, StdVec *spillers) +{ + txn_spill_spillers_t *c = captures; + + for (size_t k = 0; k < spillers->len; k++) + if (STD_VEC_AT(spillers, uint64, k) == c->xid) + { + c->done = true; + return; + } + c->done = std_vec_try_reserve(spillers, 1, sizeof(uint64), _Alignof(uint64)); + if (c->done) + std_vec_push_Checked(spillers, &c->xid, sizeof(uint64), _Alignof(uint64)); +} + +/* txn_spill_spilled_free records, in the startup process, an xid whose free space map spill it listed. */ +bool +txn_spill_spilled_free(HostIoIo *io, uint64 xid, HostError *error) +{ + txn_spill_spillers_t captures = {xid}; + + txn_spill_spillers_locked(io, txn_spill_spilled_free_closure_0, &captures); + if (!captures.done) + *error = host_Error_out_of_memory(); + return captures.done; +} + +/* txn_spill_redo_aborted_closure_0 is redo_aborted's step on FREE_SPILLERS: the xid taken out, done when it was there. */ +static void +txn_spill_redo_aborted_closure_0(void *captures, StdVec *spillers) +{ + txn_spill_spillers_t *c = captures; + + for (size_t k = 0; k < spillers->len; k++) + if (STD_VEC_AT(spillers, uint64, k) == c->xid) + { + uint64 taken; + + std_vec_swap_remove(spillers, k, &taken, sizeof(uint64), _Alignof(uint64)); + c->done = true; + return; + } +} + +/* txn_spill_marked_t is the edit's captures that marks a transaction's free space map spills: the xid and the end. */ +typedef struct txn_spill_marked_t +{ + uint64 xid; + uint64 end; +} txn_spill_marked_t; + +/* txn_spill_marked_closure_0 marks each of a transaction's free space map spills with an abort record's end. */ +static bool +txn_spill_marked_closure_0(void *captures, HostSharedLongIo *io, StdVec *files, HostError *error) +{ + const txn_spill_marked_t *c = captures; + + for (size_t k = 0; k < files->len; k++) + { + TxnSpillFile *file = &STD_VEC_AT(files, TxnSpillFile, k); + + if (file->kind == TXN_RUNS_KIND_SPILLED_FREE && file->xid == c->xid) + file->passed = c->end; + } + return true; +} + +/* txn_spill_redo_aborted_frame_0_t is redo_aborted's state across its pad: the swept states, a state opened, and its files, while live. */ +typedef struct txn_spill_redo_aborted_frame_0_t +{ + HostIoIo *io; + txn_spill_marked_t marked; + HostError *error; + bool ok; + /* swept is a Vec<(u64, u64)> of the swept states' locations. */ + StdVec swept; + bool swept_live; + HostSharedHeld held; + bool held_live; + txn_TupleVecFileU64_t listed; + bool listed_live; +} txn_spill_redo_aborted_frame_0_t; + +/* txn_spill_redo_aborted_pad_0 is redo_aborted's pad: the files, the state, and the swept list, while live. */ +static void +txn_spill_redo_aborted_pad_0(void *frame_) +{ + txn_spill_redo_aborted_frame_0_t *frame = frame_; + + if (frame->listed_live) + { + frame->listed_live = false; + std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + } + if (frame->held_live) + { + frame->held_live = false; + host_shared_Held_drop(&frame->held); + } + if (frame->swept_live) + { + frame->swept_live = false; + std_vec_free_Checked(&frame->swept, sizeof(std_TupleU64U64_t), _Alignof(std_TupleU64U64_t)); + } +} + +/* txn_spill_redo_aborted_body_0 is redo_aborted's body, for a transaction spilled_free recorded. */ +static void +txn_spill_redo_aborted_body_0(void *frame_) +{ + txn_spill_redo_aborted_frame_0_t *frame = frame_; + + if (!host_shared_sweep(HOST_SHARED_IO_WITHIN(frame->io), &TXN_STATE_STATE_POOL_OPS, TXN_STATE_SPILLS_INDEX, + &frame->swept, frame->error)) + return; + frame->swept_live = true; + for (size_t k = 0; k < frame->swept.len; k++) + { + bool some; + bool spilled = false; + HostStorage storage; + TxnStatePersistence persistence; + HostSharedRecord record; + + if (!host_shared_open(HOST_SHARED_IO_WITHIN(frame->io), &TXN_STATE_STATE_POOL_OPS, + STD_VEC_AT(&frame->swept, std_TupleU64U64_t, k).f0, &some, &frame->held, frame->error)) + { + txn_spill_redo_aborted_pad_0(frame); + return; + } + if (!some) + continue; + frame->held_live = true; + record = host_shared_Held_record(&frame->held); + if (txn_state_described(record, &storage, &persistence)) + { + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, frame->io, record, &storage, &frame->listed, + frame->error)) + { + txn_spill_redo_aborted_pad_0(frame); + return; + } + frame->listed_live = true; + for (size_t j = 0; j < frame->listed.f0.len && !spilled; j++) + { + const TxnSpillFile *file = &STD_VEC_AT(&frame->listed.f0, TxnSpillFile, j); + + spilled = file->kind == TXN_RUNS_KIND_SPILLED_FREE && file->xid == frame->marked.xid; + } + frame->listed_live = false; + std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (spilled && !txn_spill_changed(&TXN_SPILL_STATE_HOME_OPS, frame->io, record, &storage, + txn_spill_marked_closure_0, &frame->marked, frame->error)) + { + txn_spill_redo_aborted_pad_0(frame); + return; + } + } + frame->held_live = false; + host_shared_Held_drop(&frame->held); + } + frame->ok = true; + txn_spill_redo_aborted_pad_0(frame); +} + +/* + * txn_spill_redo_aborted marks a replayed aborted transaction's free space + * map spills with its abort record's end, as its abort marked them on its + * primary, so a standby's folds pass them where its primary's do; it sweeps + * the states that list spills only for a transaction spilled_free recorded. + */ +bool +txn_spill_redo_aborted(HostIoIo *io, uint64 xid, HostLsn end, HostError *error) +{ + txn_spill_spillers_t known = {xid}; + txn_spill_redo_aborted_frame_0_t frame = {.io = io, .marked = {xid, end}, .error = error}; + + txn_spill_spillers_locked(io, txn_spill_redo_aborted_closure_0, &known); + if (!known.done) + return true; + std_landing(&frame, txn_spill_redo_aborted_body_0, txn_spill_redo_aborted_pad_0); + return frame.ok; +} + /* txn_spill_relieved_own_closure_0 is relieved_own's step on REPLAYED: the tree a replayed transaction wrote the most to, the last of equals as max_by_key takes it. */ static void txn_spill_relieved_own_closure_0(void *captures, StdVec *replayed) @@ -2172,8 +2768,8 @@ txn_spill_unreplayed(HostIoIo *io, const HostStorage *storage, TxnWrittenOf of, } static bool txn_spill_spilled_from(const TxnSpillHomeOps *home, HostIoIo *io, const HostSharedHeld *held, - const HostStorage *storage, TxnWrittenOf of, uint64 xid, bool *result, - HostError *error); + const HostStorage *storage, TxnWrittenOf of, uint64 xid, bool oldest, + bool *result, HostError *error); /* txn_spill_spilled_turn_captures_t is spilled_turn's step's captures: the tree, the xid, and the answer. */ typedef struct txn_spill_spilled_turn_captures_t @@ -2202,12 +2798,10 @@ txn_spill_spilled_turn_closure_0(void *captures, const TxnSpillHomeOps *home, co { txn_spill_spilled_turn_captures_t *c = captures; - if (c->of == TXN_WRITTEN_OF_IDENTITY) - return txn_spill_spilled_turn_none(c, error); - return txn_spill_spilled_from(home, c->io, held, c->storage, c->of, c->xid, c->result, error); + return txn_spill_spilled_from(home, c->io, held, c->storage, c->of, c->xid, false, c->result, error); } -/* txn_spill_spilled_turn spills a storage's tree's versions of a transaction from the tree's home, by a process whose turn it is; a database's identity tree never spills. */ +/* txn_spill_spilled_turn spills a storage's tree's versions of a transaction from the tree's home, by a process whose turn it is. */ static bool txn_spill_spilled_turn(HostIoIo *io, const HostStorage *storage, TxnWrittenOf of, uint64 xid, bool *result, HostError *error) @@ -2340,13 +2934,10 @@ txn_spill_removed_keys(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedReco return frame.ok; } -/* txn_spill_tid returns the tuple id a row version's record holds, the row a put, an add, an added row, or an update names. */ +/* txn_spill_op_tid returns the tuple id a row version's op holds, the row a put, an add, an added row, or an update names. */ static bool -txn_spill_tid(const uint8 *record, size_t record_len, uint64 *result) +txn_spill_op_tid(TxnMemtableVersionOpRef op, uint64 *result) { - TxnMemtableVersionRecord stored = txn_memtable_version_Record_of(record, record_len); - TxnMemtableVersionOpRef op = txn_memtable_version_Record_op(&stored); - switch (op.tag) { case TXN_MEMTABLE_VERSION_OP_REF_PUT: @@ -2366,6 +2957,15 @@ txn_spill_tid(const uint8 *record, size_t record_len, uint64 *result) } } +/* txn_spill_tid returns the tuple id a row version's record holds, as op_tid gives it. */ +static bool +txn_spill_tid(const uint8 *record, size_t record_len, uint64 *result) +{ + TxnMemtableVersionRecord stored = txn_memtable_version_Record_of(record, record_len); + + return txn_spill_op_tid(txn_memtable_version_Record_op(&stored), result); +} + /* txn_spill_numbered returns the number of a transaction's next spill file of a storage's kind: one past its newest listed. */ static bool txn_spill_numbered(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, const HostStorage *storage, @@ -2396,6 +2996,12 @@ typedef struct txn_spill_from_frame_0_t const HostStorage *storage; TxnWrittenOf of; uint64 xid; + + /* + * oldest is the budget's committed arm's spill, of keys whose oldest + * versions are the transaction's alone. + */ + bool oldest; bool *result; HostError *error; bool ok; @@ -2463,18 +3069,38 @@ txn_spill_from_pad_0(void *frame_) } } -/* txn_spill_from_closure_0 is spilled_from's walk's step: a key whose chain holds only the transaction's versions written, its tuple ids and its handles taken. */ +/* + * txn_spill_from_closure_0 is spilled_from's walk's step: of a key whose + * chain holds only the transaction's versions, or, for the committed arm, + * whose oldest versions are the transaction's alone, those versions written, + * their tuple ids and their handles taken, so every spilled version stays + * older than the versions the chain keeps. + */ static bool txn_spill_from_closure_0(void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *records, - size_t records_len, const txn_TupleHandleU64_t *handles, size_t handles_len, + size_t records_len, const txn_TupleHandleU64Bool_t *handles, size_t handles_len, HostError *error) { txn_spill_from_frame_0_t *frame = captures; txn_TupleVecU8VecHandle_t took; + size_t below = 0; + size_t kept = 0; - for (size_t k = 0; k < handles_len; k++) - if (handles[k].f1 != frame->xid) + for (size_t k = handles_len; k > 0 && below == 0; k--) + if (handles[k - 1].f1 != frame->xid) + below = k; + if (below == handles_len || (below > 0 && !frame->oldest)) + return true; + for (size_t k = 0; k < below; k++) + { + if (handles[k].f1 == frame->xid) return true; + kept += !handles[k].f2; + } + records += kept; + records_len -= kept; + handles += below; + handles_len -= below; if (records_len > 0) { frame->records.len = 0; @@ -2484,7 +3110,7 @@ txn_spill_from_closure_0(void *captures, HostIoIo *io, const uint8 *key, size_t std_vec_push_Checked(&frame->records, &record, sizeof(record), _Alignof(StdBytes)); } - if (!txn_runs_Writing_key(frame->writing, io, key, key_len, frame->records.ptr, frame->records.len, error)) + if (!txn_runs_Writing_key(frame->writing, HOST_IO_IO_IO_DOORS(io), key, key_len, frame->records.ptr, frame->records.len, error)) return false; } for (size_t k = 0; frame->kind == TXN_RUNS_KIND_SPILLED_ROWS && k < records_len; k++) @@ -2515,34 +3141,52 @@ txn_spill_from_closure_0(void *captures, HostIoIo *io, const uint8 *key, size_t return true; } -static bool txn_spill_tupled(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, +static bool txn_spill_tupled(const TxnSpillHomeOps *home, HostIoIoDoors io, HostSharedRecord record, const HostStorage *storage, const StdPathBuf *dir, uint64 xid, uint32 number, StdVec *tuples, TxnSpillFile *result_0, StdVec *result_1, HostError *error); static bool txn_spill_merged(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, const HostStorage *storage, uint64 xid, TxnRunsKind kind, TxnRunsTail tail, HostError *error); -/* txn_spill_from_ended ends a spill that took its tree's count down, as every way out of spilled_from does. */ +/* txn_spill_from_ended ends a spill that took its tree's count down, as every way out of spilled_from does but the committed arm's, which leaves the count to its transaction's spills. */ static bool txn_spill_from_ended(txn_spill_from_frame_0_t *frame) { + if (frame->oldest) + return true; if (!txn_written_spilled(frame->io, frame->storage, frame->of, frame->error)) return false; txn_spill_unreplayed(frame->io, frame->storage, frame->of, frame->xid); return true; } -/* txn_spill_from_listed pends, unchains, and lists a spill's files, then settles its slot and merges the transaction's files. */ +/* + * txn_spill_from_listed pends, unchains, and lists a spill's files, then + * settles its slot and merges the transaction's files, but the committed + * arm's; the startup process records a free space map's spiller. + */ static bool txn_spill_from_listed(txn_spill_from_frame_0_t *frame, HostSharedRecord record, const TxnSpillFile *listing, size_t listing_len, TxnRunsTail tail) { txn_TupleU64KindU32OptionU32_t none = {0}; + if (!txn_spill_take_began(frame->home, frame->io, record, frame->error)) + return false; if (!txn_spill_pend(frame->home, frame->io, record, frame->storage, &listing[0], listing_len > 1, false, 0, - frame->error) || + frame->error)) + { + HostError ended; + + if (!txn_spill_take_ended(frame->home, frame->io, &record, &ended)) + host_Error_drop(&ended); + return false; + } + if ((frame->kind == TXN_RUNS_KIND_SPILLED_FREE && host_wal_stand(HOST_IO_IO_IO_DOORS(frame->io)).recovering && + !txn_spill_spilled_free(frame->io, frame->xid, frame->error)) || !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-spill-pended", frame->error) || !txn_spill_removed_keys(frame->home, frame->io, record, frame->of, &frame->taken, frame->error) || + !txn_spill_take_ended(frame->home, frame->io, &record, frame->error) || (frame->tids_live && !frame->home->untupled(frame->io, record, frame->tids.ptr, frame->tids.len, frame->error)) || !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-spill-linking", frame->error) || @@ -2551,9 +3195,10 @@ txn_spill_from_listed(txn_spill_from_frame_0_t *frame, HostSharedRecord record, !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-spill-linked", frame->error) || !txn_spill_settled(frame->home, frame->io, &record, frame->error)) return false; - return txn_spill_merged(frame->home, frame->io, record, frame->storage, frame->xid, frame->kind, tail, - frame->error) && - txn_spill_from_ended(frame); + return frame->oldest || + (txn_spill_merged(frame->home, frame->io, record, frame->storage, frame->xid, frame->kind, tail, + frame->error) && + txn_spill_from_ended(frame)); } /* txn_spill_from_body_0 is spilled_from's body. */ @@ -2569,7 +3214,7 @@ txn_spill_from_body_0(void *frame_) uint32 number; bool some; RepoRunFile run; - TxnSpillFile listing[2]; + TxnSpillFile listing[2] = {0}; *frame->result = false; if (!frame->home->kind(frame->io, frame->held, frame->of, &frame->kind, &tail, frame->error) || @@ -2595,6 +3240,8 @@ txn_spill_from_body_0(void *frame_) txn_spill_from_pad_0(frame); return; } + if (frame->oldest) + number = frame->home->ended_number(record, number); frame->writing = txn_runs_Writing_spill(frame->dir.bytes.ptr, frame->dir.bytes.len, &names, tail, number); frame->writing_live = true; std_vec_new(&frame->taken, sizeof(txn_TupleVecU8VecHandle_t), _Alignof(txn_TupleVecU8VecHandle_t)); @@ -2617,7 +3264,7 @@ txn_spill_from_body_0(void *frame_) txn_spill_Filter_add(&frame->filter, key->ptr, key->len); } - if (!txn_runs_Writing_end(frame->writing, frame->io, frame->filter.words.ptr, frame->filter.words.len, &some, + if (!txn_runs_Writing_end(frame->writing, HOST_IO_IO_IO_DOORS(frame->io), frame->filter.words.ptr, frame->filter.words.len, &some, &run, frame->error)) { txn_spill_from_pad_0(frame); @@ -2639,7 +3286,7 @@ txn_spill_from_body_0(void *frame_) if (frame->tuples.len > 0) { frame->tuples_live = false; - if (!txn_spill_tupled(frame->home, frame->io, record, frame->storage, &frame->dir, frame->xid, number, + if (!txn_spill_tupled(frame->home, HOST_IO_IO_IO_DOORS(frame->io), record, frame->storage, &frame->dir, frame->xid, number, &frame->tuples, &listing[1], &frame->tids, frame->error)) { txn_spill_from_pad_0(frame); @@ -2660,19 +3307,24 @@ txn_spill_from_body_0(void *frame_) /* * txn_spill_spilled_from spills a tree's versions of a transaction from its * home: it walks the tree and writes, in key order, each key whose chain - * holds only the transaction's versions to a new spill file with its filter, - * and a row spill's tuple ids to a tuple-id file of its number; it names the - * files in the process's slot, takes the versions out of the memtable, lists - * the files in the storage's manifest, clears the slot, and merges the - * transaction's files. It zeroes the tree's count, and reports whether it - * spilled a key. + * holds only the transaction's versions, or, for the budget's committed arm + * (oldest), the oldest versions in the memtable that are the transaction's + * alone, to a new spill file with its filter, and a row spill's tuple ids to + * a tuple-id file of its number; it names the files in the process's slot, + * takes the versions out of the memtable, lists the files in the storage's + * manifest, clears the slot, and merges the transaction's files. It zeroes + * the tree's count, and reports whether it spilled a key. The committed arm + * takes its file's number through the home's ended_number, since a fold's + * residual numbers the same transaction's files, and neither merges nor + * zeroes a count. */ static bool txn_spill_spilled_from(const TxnSpillHomeOps *home, HostIoIo *io, const HostSharedHeld *held, - const HostStorage *storage, TxnWrittenOf of, uint64 xid, bool *result, HostError *error) + const HostStorage *storage, TxnWrittenOf of, uint64 xid, bool oldest, bool *result, + HostError *error) { txn_spill_from_frame_0_t frame = {.home = home, .io = io, .held = held, .storage = storage, .of = of, .xid = xid, - .result = result, .error = error}; + .oldest = oldest, .result = result, .error = error}; std_landing(&frame, txn_spill_from_body_0, txn_spill_from_pad_0); return frame.ok; @@ -2716,7 +3368,7 @@ txn_spill_TupleU64VecU8_lt(void *captures, const void *a, const void *b) typedef struct txn_spill_tupled_frame_0_t { const TxnSpillHomeOps *home; - HostIoIo *io; + HostIoIoDoors io; HostSharedRecord record; const HostStorage *storage; const StdPathBuf *dir; @@ -2846,12 +3498,11 @@ txn_spill_tupled_body_0(void *frame_) txn_spill_tupled_pad_0(frame); return; } - frame->result_0->xid = frame->xid; - frame->result_0->kind = TXN_RUNS_KIND_SPILLED_TUPLES; - frame->result_0->number = frame->number; - frame->result_0->len = run.len; - frame->result_0->hash = run.hash; - frame->result_0->bits = txn_spill_Filter_bits(&frame->filter); + *frame->result_0 = (TxnSpillFile) + { + .xid = frame->xid, .kind = TXN_RUNS_KIND_SPILLED_TUPLES, .number = frame->number, .len = run.len, + .hash = run.hash, .bits = txn_spill_Filter_bits(&frame->filter) + }; frame->result_1_live = false; frame->ok = true; txn_spill_tupled_pad_0(frame); @@ -2864,7 +3515,7 @@ txn_spill_tupled_body_0(void *frame_) * with its tids. It takes the tuples. */ static bool -txn_spill_tupled(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, const HostStorage *storage, +txn_spill_tupled(const TxnSpillHomeOps *home, HostIoIoDoors io, HostSharedRecord record, const HostStorage *storage, const StdPathBuf *dir, uint64 xid, uint32 number, StdVec *tuples, TxnSpillFile *result_0, StdVec *result_1, HostError *error) { @@ -3030,14 +3681,422 @@ txn_spill_helped(HostIoIo *io, bool *result, HostError *error) return frame.ok; } -/* txn_spill_relieved spills, for a writer that found the delta full, its own trees, or else another transaction's, and reports whether one spilled a key. */ -bool -txn_spill_relieved(HostIoIo *io, bool *result, HostError *error) +/* TxnSpillCommittedState is a state the committed arm may spill: its delta's bytes, the state held, and its storage. */ +typedef struct TxnSpillCommittedState { - if (!txn_spill_relieved_own(io, result, error)) - return false; - return *result || txn_spill_helped(io, result, error); -} + uint64 bytes; + HostSharedHeld held; + HostStorage storage; +} TxnSpillCommittedState; + +/* TxnSpillOldest is a writer of a chain's oldest memtable version that committed: its commit's end, its top-level xid, and the tree. */ +typedef struct TxnSpillOldest +{ + HostLsn end; + uint64 xid; + TxnWrittenOf of; +} TxnSpillOldest; + +/* txn_spill_committed_frame_0_t is committed's state across its pad: the swept locations, a state opened, and the states held, while live. */ +typedef struct txn_spill_committed_frame_0_t +{ + HostIoIo *io; + bool *result; + HostError *error; + bool ok; + /* swept is a Vec<(u64, u64)> of a sweep's locations and words. */ + StdVec swept; + bool swept_live; + HostSharedHeld held; + bool held_live; + /* states is a Vec. */ + StdVec states; + bool states_live; + /* oldest is a Vec of one state's writers. */ + StdVec oldest; + bool oldest_live; + + /* + * turned is the state whose turn the process holds, as id, while + * turn_live. + */ + HostSharedRecord turned; + uint64 id; + bool turn_live; +} txn_spill_committed_frame_0_t; + +/* txn_spill_committed_pad_0 is committed's pad: the turn taken, the writers, the states held, the state opened, and the swept list, while live. */ +static void +txn_spill_committed_pad_0(void *frame_) +{ + txn_spill_committed_frame_0_t *frame = frame_; + + if (frame->turn_live) + { + frame->turn_live = false; + txn_state_committed_given(frame->turned, frame->id); + } + if (frame->oldest_live) + { + frame->oldest_live = false; + std_vec_free_Checked(&frame->oldest, sizeof(TxnSpillOldest), _Alignof(TxnSpillOldest)); + } + if (frame->states_live) + { + frame->states_live = false; + for (size_t k = 0; k < frame->states.len; k++) + host_shared_Held_drop(&STD_VEC_AT(&frame->states, TxnSpillCommittedState, k).held); + std_vec_free_Checked(&frame->states, sizeof(TxnSpillCommittedState), _Alignof(TxnSpillCommittedState)); + } + if (frame->held_live) + { + frame->held_live = false; + host_shared_Held_drop(&frame->held); + } + if (frame->swept_live) + { + frame->swept_live = false; + std_vec_free_Checked(&frame->swept, sizeof(std_TupleU64U64_t), _Alignof(std_TupleU64U64_t)); + } +} + +/* txn_spill_committed_bytes_cmp orders committed's states by their delta's bytes, the most first. */ +static int +txn_spill_committed_bytes_cmp(const void *a, const void *b) +{ + uint64 x = ((const TxnSpillCommittedState *) a)->bytes; + uint64 y = ((const TxnSpillCommittedState *) b)->bytes; + + return x > y ? -1 : x < y; +} + +/* txn_spill_oldest_cmp orders a state's oldest writers by their commits' ends, then their xids. */ +static int +txn_spill_oldest_cmp(const void *a, const void *b) +{ + const TxnSpillOldest *x = a; + const TxnSpillOldest *y = b; + + if (x->end != y->end) + return x->end < y->end ? -1 : 1; + return x->xid < y->xid ? -1 : x->xid > y->xid; +} + +/* txn_spill_committed_oldest_t is committed_from's walk's captures: the frame and the tree walked. */ +typedef struct txn_spill_committed_oldest_t +{ + txn_spill_committed_frame_0_t *frame; + TxnWrittenOf of; +} txn_spill_committed_oldest_t; + +/* txn_spill_committed_closure_0 is committed_from's walk's step: the writer of a chain's oldest memtable version, once, when it committed. */ +static bool +txn_spill_committed_closure_0(void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *records, + size_t records_len, const txn_TupleHandleU64Bool_t *handles, size_t handles_len, + HostError *error) +{ + txn_spill_committed_oldest_t *c = captures; + StdVec *oldest = &c->frame->oldest; + TxnSpillOldest writer = {.of = c->of}; + HostXid xid; + TxnStatusCommit commit; + + if (handles_len == 0) + return true; + writer.xid = handles[handles_len - 1].f1; + for (size_t k = 0; k < oldest->len; k++) + { + const TxnSpillOldest *at = &STD_VEC_AT(oldest, TxnSpillOldest, k); + + if (at->xid == writer.xid && at->of == writer.of) + return true; + } + xid.f0 = writer.xid; + if (!txn_status_committed(io, xid, &commit, error)) + return false; + if (commit.tag != TXN_STATUS_COMMIT_AT) + return true; + writer.end = commit.u.at; + if (!std_vec_try_reserve(oldest, 1, sizeof(writer), _Alignof(TxnSpillOldest))) + { + *error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(oldest, &writer, sizeof(writer), _Alignof(TxnSpillOldest)); + return true; +} + +/* + * txn_spill_committed_from spills a storage's committed versions as + * committed does, under the process's turn at it: the writers of its rows' + * and value map's chains' oldest memtable versions that committed, the + * oldest commit first. + */ +static bool +txn_spill_committed_from(txn_spill_committed_frame_0_t *frame, const TxnSpillCommittedState *state) +{ + static const TxnWrittenOf ofs[] = {TXN_WRITTEN_OF_ROWS, TXN_WRITTEN_OF_VALUES}; + + std_vec_new(&frame->oldest, sizeof(TxnSpillOldest), _Alignof(TxnSpillOldest)); + frame->oldest_live = true; + for (size_t k = 0; k < lengthof(ofs); k++) + { + txn_spill_committed_oldest_t captures = {frame, ofs[k]}; + + if (!txn_spill_Home_copied(&TXN_SPILL_STATE_HOME_OPS, frame->io, host_shared_Held_record(&state->held), + ofs[k], txn_spill_committed_closure_0, &captures, frame->error)) + return false; + } + if (frame->oldest.len > 1) + qsort(frame->oldest.ptr, frame->oldest.len, sizeof(TxnSpillOldest), txn_spill_oldest_cmp); + for (size_t k = 0; k < frame->oldest.len && !*frame->result; k++) + { + const TxnSpillOldest *writer = &STD_VEC_AT(&frame->oldest, TxnSpillOldest, k); + + if (!txn_spill_spilled_from(&TXN_SPILL_STATE_HOME_OPS, frame->io, &state->held, &state->storage, writer->of, + writer->xid, true, frame->result, frame->error)) + return false; + } + frame->oldest_live = false; + std_vec_free_Checked(&frame->oldest, sizeof(TxnSpillOldest), _Alignof(TxnSpillOldest)); + return true; +} + +/* txn_spill_committed_body_0 is committed's body. */ +static void +txn_spill_committed_body_0(void *frame_) +{ + txn_spill_committed_frame_0_t *frame = frame_; + HostSharedMine mine; + + *frame->result = false; + if (!txn_spill_mine(frame->io, &mine, frame->error) || + !host_shared_sweep(HOST_SHARED_IO_WITHIN(frame->io), &TXN_STATE_STATE_POOL_OPS, + TXN_STATE_MEMTABLE_WORD_INDEX, &frame->swept, frame->error)) + return; + frame->id = host_shared_Mine_id(&mine); + frame->swept_live = true; + std_vec_new(&frame->states, sizeof(TxnSpillCommittedState), _Alignof(TxnSpillCommittedState)); + frame->states_live = true; + for (size_t k = 0; k < frame->swept.len; k++) + { + bool some; + TxnSpillCommittedState state; + TxnStatePersistence persistence; + + if (!host_shared_open(HOST_SHARED_IO_WITHIN(frame->io), &TXN_STATE_STATE_POOL_OPS, + STD_VEC_AT(&frame->swept, std_TupleU64U64_t, k).f0, &some, &frame->held, frame->error)) + { + txn_spill_committed_pad_0(frame); + return; + } + if (!some) + continue; + frame->held_live = true; + if (!txn_state_described(host_shared_Held_record(&frame->held), &state.storage, &persistence)) + { + frame->held_live = false; + host_shared_Held_drop(&frame->held); + continue; + } + if (!std_vec_try_reserve(&frame->states, 1, sizeof(state), _Alignof(TxnSpillCommittedState))) + { + *frame->error = host_Error_out_of_memory(); + txn_spill_committed_pad_0(frame); + return; + } + state.bytes = txn_state_delta_bytes(host_shared_Held_record(&frame->held)); + state.held = frame->held; + frame->held_live = false; + std_vec_push_Checked(&frame->states, &state, sizeof(state), _Alignof(TxnSpillCommittedState)); + } + if (frame->states.len > 1) + qsort(frame->states.ptr, frame->states.len, sizeof(TxnSpillCommittedState), txn_spill_committed_bytes_cmp); + for (size_t k = 0; k < frame->states.len && !*frame->result; k++) + { + const TxnSpillCommittedState *state = &STD_VEC_AT(&frame->states, TxnSpillCommittedState, k); + HostSharedRecord record = host_shared_Held_record(&state->held); + bool spilled; + + if (!txn_state_committed_taken(record, frame->id)) + continue; + frame->turned = record; + frame->turn_live = true; + spilled = txn_spill_committed_from(frame, state); + frame->turn_live = false; + txn_state_committed_given(record, frame->id); + if (!spilled) + { + txn_spill_committed_pad_0(frame); + return; + } + } + frame->ok = true; + txn_spill_committed_pad_0(frame); +} + +/* + * txn_spill_committed spills, for a writer the full delta refused when no + * transaction had versions of its own left to spill, the committed versions + * a fold leaves in a storage's memtable, its point held below their commits + * by a snapshot (the budget's committed arm): of the storages, the most + * delta bytes first, each whose turn at it no other process holds, its + * committed transactions' versions of its rows and its value map, the oldest + * commit first, of every key whose oldest versions in the memtable are that + * transaction's alone, the spilled versions below them older still + * (spilled_from), until one spills a key; it answers whether one did. + */ +static bool +txn_spill_committed(HostIoIo *io, bool *result, HostError *error) +{ + txn_spill_committed_frame_0_t frame = {.io = io, .result = result, .error = error}; + + std_landing(&frame, txn_spill_committed_body_0, txn_spill_committed_pad_0); + return frame.ok; +} + +/* txn_spill_identity_saved_frame_0_t is identity_saved's state across its pad: a sweep's locations, a delta opened, and the repositories found, while live. */ +typedef struct txn_spill_identity_saved_frame_0_t +{ + HostIoIo *io; + bool *result; + HostError *error; + bool ok; + /* swept is a Vec<(u64, u64)> of a sweep's locations and words. */ + StdVec swept; + bool swept_live; + HostSharedHeld held; + bool held_live; + /* found is a Vec<(u32, u32)> of repositories and their own tablespaces. */ + StdVec found; + bool found_live; +} txn_spill_identity_saved_frame_0_t; + +/* txn_spill_identity_saved_pad_0 is identity_saved's pad. */ +static void +txn_spill_identity_saved_pad_0(void *frame_) +{ + txn_spill_identity_saved_frame_0_t *frame = frame_; + + if (frame->held_live) + host_shared_Held_drop(&frame->held); + if (frame->swept_live) + std_vec_free_Checked(&frame->swept, sizeof(std_TupleU64U64_t), _Alignof(std_TupleU64U64_t)); + if (frame->found_live) + std_vec_free_Checked(&frame->found, sizeof(std_TupleU32U32_t), _Alignof(std_TupleU32U32_t)); + frame->held_live = frame->swept_live = frame->found_live = false; +} + +/* txn_spill_identity_saved_found adds the repository of each delta a sweep of one of its words finds, once. */ +static bool +txn_spill_identity_saved_found(txn_spill_identity_saved_frame_0_t *frame, size_t word) +{ + if (!host_shared_sweep(HOST_SHARED_IO_WITHIN(frame->io), &TXN_MEMTABLE_BRANCH_BRANCH_DELTAS_POOL_OPS, word, + &frame->swept, frame->error)) + return false; + frame->swept_live = true; + for (size_t k = 0; k < frame->swept.len; k++) + { + bool some; + std_TupleU32U32_t repository; + bool known = false; + + if (!host_shared_open(HOST_SHARED_IO_WITHIN(frame->io), &TXN_MEMTABLE_BRANCH_BRANCH_DELTAS_POOL_OPS, + STD_VEC_AT(&frame->swept, std_TupleU64U64_t, k).f0, &some, &frame->held, + frame->error)) + return false; + if (!some) + continue; + frame->held_live = true; + if (txn_memtable_branch_repository(host_shared_Held_record(&frame->held), &repository.f0)) + { + repository.f1 = txn_memtable_branch_own(host_shared_Held_record(&frame->held)); + for (size_t f = 0; f < frame->found.len && !known; f++) + known = STD_VEC_AT(&frame->found, std_TupleU32U32_t, f).f0 == repository.f0; + if (!known) + { + if (!std_vec_try_reserve(&frame->found, 1, sizeof(repository), _Alignof(std_TupleU32U32_t))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(&frame->found, &repository, sizeof(repository), _Alignof(std_TupleU32U32_t)); + } + } + frame->held_live = false; + host_shared_Held_drop(&frame->held); + } + frame->swept_live = false; + std_vec_free_Checked(&frame->swept, sizeof(std_TupleU64U64_t), _Alignof(std_TupleU64U64_t)); + return true; +} + +/* txn_spill_identity_saved_body_0 is identity_saved's body. */ +static void +txn_spill_identity_saved_body_0(void *frame_) +{ + txn_spill_identity_saved_frame_0_t *frame = frame_; + + *frame->result = false; + std_vec_new(&frame->found, sizeof(std_TupleU32U32_t), _Alignof(std_TupleU32U32_t)); + frame->found_live = true; + if (!txn_spill_identity_saved_found(frame, TXN_MEMTABLE_BRANCH_IDENTITY_WORD) || + !txn_spill_identity_saved_found(frame, TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_INDEX)) + { + txn_spill_identity_saved_pad_0(frame); + return; + } + for (size_t k = 0; k < frame->found.len; k++) + { + const std_TupleU32U32_t *repository = &STD_VEC_AT(&frame->found, std_TupleU32U32_t, k); + bool took; + bool cleaned; + + if (!txn_runs_save(frame->io, repository->f0, repository->f1, &took, &cleaned, frame->error)) + { + txn_spill_identity_saved_pad_0(frame); + return; + } + *frame->result = *frame->result || took; + } + frame->ok = true; + txn_spill_identity_saved_pad_0(frame); +} + +/* + * txn_spill_identity_saved is relieved's database trees' arm (bugs row 340): + * it runs item 6's save for each repository whose working state's delta + * holds identity versions or lists identity spills, whose fold takes the + * committed ones out, and answers whether a save took a version or a file + * out; a version committed after a save's point stays for the next. + */ +static bool +txn_spill_identity_saved(HostIoIo *io, bool *result, HostError *error) +{ + txn_spill_identity_saved_frame_0_t frame = {.io = io, .result = result, .error = error}; + + std_landing(&frame, txn_spill_identity_saved_body_0, txn_spill_identity_saved_pad_0); + return frame.ok; +} + +/* + * txn_spill_relieved spills, for a writer that found the delta full, its own + * trees, or else another transaction's, and reports whether one spilled a + * key; once no transaction has versions of its own left to spill, it spills + * committed versions a fold leaves (committed), and then saves the + * repositories whose identity trees hold committed versions (identity_saved). + */ +bool +txn_spill_relieved(HostIoIo *io, bool *result, HostError *error) +{ + if (!txn_spill_relieved_own(io, result, error)) + return false; + if (!*result && !txn_spill_helped(io, result, error)) + return false; + if (!*result && !txn_spill_committed(io, result, error)) + return false; + return *result || txn_spill_identity_saved(io, result, error); +} /* txn_spill_swept_frame_0_t is swept's state across its pad: the swept locations, a home opened, and the trees, while live. */ typedef struct txn_spill_swept_frame_0_t @@ -3127,16 +4186,19 @@ static void txn_spill_swept_body_0(void *frame_) { txn_spill_swept_frame_0_t *frame = frame_; - static const TxnWrittenOf state_ofs[] = {TXN_WRITTEN_OF_ROWS, TXN_WRITTEN_OF_VALUES}; + static const TxnWrittenOf state_ofs[] = {TXN_WRITTEN_OF_ROWS, TXN_WRITTEN_OF_VALUES, TXN_WRITTEN_OF_FREE}; static const TxnWrittenOf chunk_ofs[] = {TXN_WRITTEN_OF_CHUNKS}; + static const TxnWrittenOf identity_ofs[] = {TXN_WRITTEN_OF_IDENTITY}; *frame->result = false; std_vec_new(&frame->trees, sizeof(txn_TupleStorageOf_t), _Alignof(txn_TupleStorageOf_t)); frame->trees_live = true; if (!txn_spill_swept_pool(frame, &TXN_STATE_STATE_POOL_OPS, TXN_STATE_MEMTABLE_WORD_INDEX, - &TXN_SPILL_STATE_HOME_OPS, state_ofs, 2) || + &TXN_SPILL_STATE_HOME_OPS, state_ofs, lengthof(state_ofs)) || !txn_spill_swept_pool(frame, &TXN_MEMTABLE_BRANCH_BRANCH_DELTAS_POOL_OPS, - TXN_MEMTABLE_BRANCH_CHUNK_WORD_INDEX, &TXN_SPILL_BRANCH_DELTAS_HOME_OPS, chunk_ofs, 1)) + TXN_MEMTABLE_BRANCH_CHUNK_WORD_INDEX, &TXN_SPILL_BRANCH_DELTAS_HOME_OPS, chunk_ofs, 1) || + !txn_spill_swept_pool(frame, &TXN_MEMTABLE_BRANCH_BRANCH_DELTAS_POOL_OPS, TXN_MEMTABLE_BRANCH_IDENTITY_WORD, + &TXN_SPILL_IDENTITY_HOME_OPS, identity_ofs, 1)) { txn_spill_swept_pad_0(frame); return; @@ -3159,8 +4221,8 @@ txn_spill_swept_body_0(void *frame_) /* * txn_spill_swept spills a transaction's versions from every storage's - * memtable, both trees, and every repository's chunk tree, for a helper - * whose turn it is, and reports whether it spilled a key. + * memtable, its three trees, and every repository's chunk and identity + * trees, for a helper whose turn it is, and reports whether it spilled a key. */ static bool txn_spill_swept(HostIoIo *io, uint64 xid, bool *result, HostError *error) @@ -3178,13 +4240,48 @@ txn_spill_aborted_closure_0(void *captures, const TxnSpillFile *file) return file->xid == *(const uint64 *) captures; } -/* txn_spill_aborted_closure_1 is aborted's step on the home of its pending spill: the slot cleared. */ +/* txn_spill_aborted_pending_t is aborted's step's captures on the home of its pending spill: the backend's I/O and the spill's kind. */ +typedef struct txn_spill_aborted_pending_t +{ + HostIoIo *io; + TxnRunsKind kind; +} txn_spill_aborted_pending_t; + +/* + * txn_spill_aborted_closure_1 is aborted's step on the home of its pending + * spill: a free space map's spill of a storage listed from the slot, and any + * other the slot cleared. + */ static bool txn_spill_aborted_closure_1(void *captures, const TxnSpillHomeOps *home, const HostSharedHeld *held, HostError *error) { + const txn_spill_aborted_pending_t *c = captures; HostSharedRecord record = host_shared_Held_record(held); - return txn_spill_settled(home, captures, &record, error); + if (home == &TXN_SPILL_STATE_HOME_OPS && c->kind == TXN_RUNS_KIND_SPILLED_FREE) + return txn_spill_settle_pending(c->io, error); + return txn_spill_settled(home, c->io, &record, error); +} + +/* txn_spill_aborted_kept_closure_0 is aborted's edit of a storage's manifest: the transaction's files out but its free space map's, which it marks with the abort's end. */ +static bool +txn_spill_aborted_kept_closure_0(void *captures, HostSharedLongIo *io, StdVec *files, HostError *error) +{ + const txn_spill_marked_t *c = captures; + size_t kept = 0; + + for (size_t k = 0; k < files->len; k++) + { + TxnSpillFile file = STD_VEC_AT(files, TxnSpillFile, k); + + if (file.xid == c->xid && file.kind != TXN_RUNS_KIND_SPILLED_FREE) + continue; + if (file.xid == c->xid) + file.passed = c->end; + STD_VEC_AT(files, TxnSpillFile, kept++) = file; + } + files->len = kept; + return true; } /* txn_spill_aborted_captures_t is aborted's step's captures on a storage it spilled: the backend's I/O, the storage, and the xid. */ @@ -3192,18 +4289,25 @@ typedef struct txn_spill_aborted_captures_t { HostIoIo *io; const HostStorage *storage; - uint64 xid; + txn_spill_marked_t marked; } txn_spill_aborted_captures_t; -/* txn_spill_aborted_closure_2 is aborted's step on a storage it spilled: the transaction's files taken out of its manifest. */ +/* + * txn_spill_aborted_closure_2 is aborted's step on a storage it spilled: the + * transaction's files taken out of its manifest, but a storage's free space + * map's spills, which stay, marked with the abort's end. + */ static bool txn_spill_aborted_closure_2(void *captures, const TxnSpillHomeOps *home, const HostSharedHeld *held, HostError *error) { txn_spill_aborted_captures_t *c = captures; StdVec gone; + if (home == &TXN_SPILL_STATE_HOME_OPS) + return txn_spill_changed(home, c->io, host_shared_Held_record(held), c->storage, + txn_spill_aborted_kept_closure_0, &c->marked, error); if (!txn_spill_removed(home, c->io, host_shared_Held_record(held), c->storage, txn_spill_aborted_closure_0, - &c->xid, &gone, error)) + &c->marked.xid, &gone, error)) return false; std_vec_free_Checked(&gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); return true; @@ -3213,10 +4317,13 @@ txn_spill_aborted_closure_2(void *captures, const TxnSpillHomeOps *home, const H * txn_spill_aborted takes an aborted transaction's spills out of the * manifests of the storages it spilled, and clears the process's slot when * it names one of its spills; a storage whose state is gone took its spills - * with it. + * with it. A free space map's spill stays, listed from the slot when the + * slot names it, since an aborted take keeps its place until a fold takes + * the file, marked with the abort's end, the point a fold passes it at. */ bool -txn_spill_aborted(HostIoIo *io, uint64 xid, const HostStorage *storages, size_t storages_len, HostError *error) +txn_spill_aborted(HostIoIo *io, uint64 xid, const HostStorage *storages, size_t storages_len, HostLsn end, + HostError *error) { bool some; TxnSpillPended pended; @@ -3225,14 +4332,16 @@ txn_spill_aborted(HostIoIo *io, uint64 xid, const HostStorage *storages, size_t return false; if (some && pended.xid == xid) { - if (!txn_spill_at_found(io, &pended.storage, txn_spill_aborted_closure_1, io, &some, error)) + txn_spill_aborted_pending_t pending = {io, pended.kind}; + + if (!txn_spill_at_found(io, &pended.storage, txn_spill_aborted_closure_1, &pending, &some, error)) return false; if (!some && !txn_spill_settled(&TXN_SPILL_STATE_HOME_OPS, io, NULL, error)) return false; } for (size_t k = 0; k < storages_len; k++) { - txn_spill_aborted_captures_t captures = {io, &storages[k], xid}; + txn_spill_aborted_captures_t captures = {io, &storages[k], {xid, end}}; if (!txn_spill_at_found(io, &storages[k], txn_spill_aborted_closure_2, &captures, &some, error)) return false; @@ -3335,9 +4444,31 @@ txn_spill_Source_front(TxnSpillSource *self, HostIoIo *io, const TxnRunsNames *n return true; } -/* txn_spill_joined_frame_0_t is joined's state across its pad: the sources, the filter, the writing, the least key, and its versions, while live. */ -typedef struct txn_spill_joined_frame_0_t -{ +static bool txn_spill_ends_hold(const StdVec *ends, uint64 lsn); + +/* + * TxnSpillLeft is what a join leaves out of a carry its primary unlisted: + * the ends of the carry's records, ascending; the count of the versions it + * dropped; and the tuple ids of the rows it dropped, which a rows join adds + * and a tuple-id join of the same files leaves out, ascending by then. + */ +typedef struct TxnSpillLeft +{ + const StdVec *ends; + + /* + * carried is the carried rows' tuple ids, ascending, whose free space + * takes, which carry no LSN, a free space run leaves out, or NULL. + */ + const StdVec *carried; + uint64 dropped; + /* tids is a Vec. */ + StdVec tids; +} TxnSpillLeft; + +/* txn_spill_joined_frame_0_t is joined's state across its pad: the sources, the filter, the writing, the least key, and its versions, while live. */ +typedef struct txn_spill_joined_frame_0_t +{ HostIoIo *io; const StdPathBuf *dir; const TxnRunsNames *names; @@ -3345,6 +4476,12 @@ typedef struct txn_spill_joined_frame_0_t const TxnSpillFile *files; size_t files_len; uint32 number; + + /* + * left is what the join leaves out of an unlisted carry, or NULL for + * none. + */ + TxnSpillLeft *left; TxnSpillFile *result; HostError *error; bool ok; @@ -3523,18 +4660,45 @@ txn_spill_joined_key(txn_spill_joined_frame_0_t *frame) } if (frame->names->kind == TXN_RUNS_KIND_SPILLED_TUPLES) std_vec_truncate(&frame->versions, 1, sizeof(StdVec), txn_spill_vec_u8_drop); + if (frame->left != NULL && frame->names->kind == TXN_RUNS_KIND_SPILLED_TUPLES && frame->least.len == 8 && + txn_spill_ends_hold(&frame->left->tids, std_u64_from_be_bytes(frame->least.ptr))) + std_vec_truncate(&frame->versions, 0, sizeof(StdVec), txn_spill_vec_u8_drop); frame->records.len = 0; for (size_t k = 0; k < frame->versions.len; k++) { const StdVec *version = &STD_VEC_AT(&frame->versions, StdVec, k); StdBytes record = {version->ptr, version->len}; + if (frame->left != NULL && frame->names->kind != TXN_RUNS_KIND_SPILLED_TUPLES) + { + TxnMemtableVersionRecord stored = txn_memtable_version_Record_of(version->ptr, version->len); + uint64 lsn; + uint64 tid; + + if (txn_memtable_version_Record_lsn(&stored, &lsn) && txn_spill_ends_hold(frame->left->ends, lsn)) + { + frame->left->dropped++; + if (txn_spill_tid(version->ptr, version->len, &tid)) + std_vec_push_Checked(&frame->left->tids, &tid, sizeof(uint64), _Alignof(uint64)); + continue; + } + tid = txn_memtable_map_taken(txn_memtable_version_Record_op(&stored)).f0; + if (frame->left->carried != NULL && frame->names->kind == TXN_RUNS_KIND_SPILLED_FREE && tid != 0 && + txn_spill_ends_hold(frame->left->carried, tid)) + { + frame->left->dropped++; + continue; + } + } std_vec_push_Checked(&frame->records, &record, sizeof(record), _Alignof(StdBytes)); } - if (!txn_runs_Writing_key(frame->writing, frame->io, frame->least.ptr, frame->least.len, frame->records.ptr, - frame->records.len, frame->error)) - return false; - txn_spill_Filter_add(&frame->filter, frame->least.ptr, frame->least.len); + if (frame->records.len > 0) + { + if (!txn_runs_Writing_key(frame->writing, HOST_IO_IO_IO_DOORS(frame->io), frame->least.ptr, frame->least.len, + frame->records.ptr, frame->records.len, frame->error)) + return false; + txn_spill_Filter_add(&frame->filter, frame->least.ptr, frame->least.len); + } frame->versions_live = false; txn_spill_versions_drop(&frame->versions); return true; @@ -3601,24 +4765,33 @@ txn_spill_joined_body_0(void *frame_) return; } } - if (!txn_runs_Writing_end(frame->writing, frame->io, frame->filter.words.ptr, frame->filter.words.len, &some, &run, + if (!txn_runs_Writing_end(frame->writing, HOST_IO_IO_IO_DOORS(frame->io), frame->filter.words.ptr, frame->filter.words.len, &some, &run, frame->error)) { txn_spill_joined_pad_0(frame); return; } + if (!some && frame->left != NULL) + { + *frame->result = (TxnSpillFile) + { + 0 + }; + frame->ok = true; + txn_spill_joined_pad_0(frame); + return; + } if (!some) { *frame->error = txn_spill_internal("a merge of spills wrote no key"); txn_spill_joined_pad_0(frame); return; } - frame->result->xid = frame->files_len > 0 ? frame->files[0].xid : 0; - frame->result->kind = frame->names->kind; - frame->result->number = frame->number; - frame->result->len = run.len; - frame->result->hash = run.hash; - frame->result->bits = txn_spill_Filter_bits(&frame->filter); + *frame->result = (TxnSpillFile) + { + .xid = frame->files_len > 0 ? frame->files[0].xid : 0, .kind = frame->names->kind, .number = frame->number, + .len = run.len, .hash = run.hash, .bits = txn_spill_Filter_bits(&frame->filter) + }; frame->ok = true; txn_spill_joined_pad_0(frame); } @@ -3627,14 +4800,18 @@ txn_spill_joined_body_0(void *frame_) * txn_spill_joined streams spill files of one kind into one new file of a * number with their header's tail, in key order, a key's versions the newer * file's first, a tuple id's key the newest file's alone, with a filter of - * its keys, synced, and returns it. + * its keys, synced, and returns it; given what to leave out of an unlisted + * carry, it leaves out each version whose LSN its ends hold, adding the + * dropped rows' tuple ids, or for tuple ids each one it holds, and a key left + * with none, and answers a file of no length when it leaves out every one. */ static bool txn_spill_joined(HostIoIo *io, const StdPathBuf *dir, const TxnRunsNames *names, TxnRunsTail tail, - const TxnSpillFile *files, size_t files_len, uint32 number, TxnSpillFile *result, HostError *error) + const TxnSpillFile *files, size_t files_len, uint32 number, TxnSpillLeft *left, + TxnSpillFile *result, HostError *error) { txn_spill_joined_frame_0_t frame = {.io = io, .dir = dir, .names = names, .tail = tail, .files = files, - .files_len = files_len, .number = number, .result = result, .error = error}; + .files_len = files_len, .number = number, .left = left, .result = result, .error = error}; std_landing(&frame, txn_spill_joined_body_0, txn_spill_joined_pad_0); return frame.ok; @@ -3719,7 +4896,7 @@ txn_spill_merged_step(txn_spill_merged_frame_0_t *frame, const TxnRunsNames *nam size_t taken = 1; uint32 number = 0; uint32 lowest; - TxnSpillFile listing[2]; + TxnSpillFile listing[2] = {0}; size_t listing_len = 1; TxnRunsTail none = {.tag = TXN_RUNS_TAIL_NONE}; txn_TupleU64KindU32OptionU32_t replacing; @@ -3752,8 +4929,8 @@ txn_spill_merged_step(txn_spill_merged_frame_0_t *frame, const TxnRunsNames *nam lowest = number; for (size_t k = 0; k < taken; k++) lowest = Min(lowest, STD_VEC_AT(&frame->own, TxnSpillFile, k).number); - if (!txn_spill_joined(frame->io, &frame->dir, names, frame->tail, frame->own.ptr, taken, number, &listing[0], - frame->error)) + if (!txn_spill_joined(frame->io, &frame->dir, names, frame->tail, frame->own.ptr, taken, number, NULL, + &listing[0], frame->error)) return false; txn_spill_merged_mine(frame, TXN_RUNS_KIND_SPILLED_TUPLES, &frame->named); frame->named_live = true; @@ -3765,7 +4942,7 @@ txn_spill_merged_step(txn_spill_merged_frame_0_t *frame, const TxnRunsNames *nam const TxnSpillFile *tuples = &STD_VEC_AT(&frame->named, TxnSpillFile, j); bool merged = false; - for (size_t k = 0; k < taken && !merged; k++) + for (size_t k = 0; frame->kind == TXN_RUNS_KIND_SPILLED_ROWS && k < taken && !merged; k++) merged = STD_VEC_AT(&frame->own, TxnSpillFile, k).number == tuples->number; if (merged) STD_VEC_AT(&frame->named, TxnSpillFile, kept++) = *tuples; @@ -3775,7 +4952,7 @@ txn_spill_merged_step(txn_spill_merged_frame_0_t *frame, const TxnRunsNames *nam if (frame->named.len > 0) { if (!txn_spill_joined(frame->io, &frame->dir, tuple_names, none, frame->named.ptr, frame->named.len, number, - &listing[1], frame->error)) + NULL, &listing[1], frame->error)) return false; listing_len = 2; } @@ -3835,10 +5012,11 @@ txn_spill_merged_body_0(void *frame_) * txn_spill_merged merges a transaction's newest spill file of a storage's * kind with its earlier files newest first while each is no larger than the * files merged so far, as an LSM's tiers merge: it joins them into one new - * file, and the tuple-id files of the same numbers into one of the merged - * file's number, pends the merged file in the process's slot with the lowest - * number it replaces, lists it in place of the files it joined, and clears - * the slot, until the newest file has no earlier file to take. + * file, and for a merge of rows the tuple-id files of the same numbers into + * one of the merged file's number, pends the merged file in the process's + * slot with the lowest number it replaces, lists it in place of the files it + * joined, and clears the slot, until the newest file has no earlier file to + * take. */ static bool txn_spill_merged(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, const HostStorage *storage, @@ -3875,23 +5053,24 @@ txn_spill_xids_closure_0(void *captures, const TxnSpillFile *file) return txn_spill_has_xid(captures, file->xid); } -/* txn_spill_folded_frame_0_t is folded's state across its pad: the files listed, the xids applied, the counts, a file's path and bytes and decoding, and the files removed, while live. */ -typedef struct txn_spill_folded_frame_0_t +/* txn_spill_entered reports whether a spill file's kind counts its versions at its transaction's entry: rows, entries, and value and free space map versions. */ +static bool +txn_spill_entered(TxnRunsKind kind) +{ + return kind == TXN_RUNS_KIND_SPILLED_ROWS || kind == TXN_RUNS_KIND_SPILLED_ENTRIES || + kind == TXN_RUNS_KIND_SPILLED_VALUES || kind == TXN_RUNS_KIND_SPILLED_FREE; +} + +/* txn_spill_file_counted_frame_0_t is file_counted's state across its pad: the file's path, its bytes, and their decoding, while live. */ +typedef struct txn_spill_file_counted_frame_0_t { HostIoIo *io; HostSharedRecord state; - HostLsn point; + const HostStorage *storage; + const TxnSpillFile *file; + uint64 *result; HostError *error; bool ok; - HostStorage storage; - txn_TupleVecFileU64_t listed; - bool listed_live; - /* applied is a Vec. */ - StdVec applied; - bool applied_live; - /* counts is a Vec<(File, u64)>. */ - StdVec counts; - bool counts_live; StdPathBuf path; bool path_live; StdString name; @@ -3900,22 +5079,14 @@ typedef struct txn_spill_folded_frame_0_t bool bytes_live; TxnRunsDecoded decoded; bool decoded_live; - /* gone is a Vec. */ - StdVec gone; - bool gone_live; -} txn_spill_folded_frame_0_t; +} txn_spill_file_counted_frame_0_t; -/* txn_spill_folded_pad_0 is folded's pad: what it holds, while live. */ +/* txn_spill_file_counted_pad_0 is file_counted's pad: what it holds, while live. */ static void -txn_spill_folded_pad_0(void *frame_) +txn_spill_file_counted_pad_0(void *frame_) { - txn_spill_folded_frame_0_t *frame = frame_; + txn_spill_file_counted_frame_0_t *frame = frame_; - if (frame->gone_live) - { - frame->gone_live = false; - std_vec_free_Checked(&frame->gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - } if (frame->decoded_live) { frame->decoded_live = false; @@ -3936,6 +5107,114 @@ txn_spill_folded_pad_0(void *frame_) frame->path_live = false; std_vec_free_Checked(&frame->path.bytes, 1, 1); } +} + +/* txn_spill_file_counted_body_0 is file_counted's body. */ +static void +txn_spill_file_counted_body_0(void *frame_) +{ + txn_spill_file_counted_frame_0_t *frame = frame_; + TxnRunsNames names; + ChunksErrorError failed; + StdPathBuf joined; + + if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, frame->state, frame->storage, frame->file->kind, + frame->file->xid, &names, frame->error)) + return; + txn_runs_spill_directory(frame->storage->database, txn_state_own(frame->state), &frame->path); + frame->path_live = true; + txn_runs_spill_name(&names, frame->file->number, &frame->name); + frame->name_live = true; + std_path_join(frame->path.bytes.ptr, frame->path.bytes.len, (const char *) frame->name.ptr, frame->name.len, + &joined); + std_vec_free_Checked(&frame->path.bytes, 1, 1); + frame->path = joined; + if (!chunks_file_read_file(HOST_IO_IO_READ_DOORS(frame->io), frame->path.bytes.ptr, frame->path.bytes.len, + &frame->bytes, &failed)) + { + *frame->error = txn_base_storage(&failed); + txn_spill_file_counted_pad_0(frame); + return; + } + frame->bytes_live = true; + if (!txn_runs_decode(frame->bytes.ptr, frame->bytes.len, &names, frame->file->hash, &frame->decoded, + frame->error)) + { + txn_spill_file_counted_pad_0(frame); + return; + } + frame->decoded_live = true; + *frame->result = txn_spill_counted(&frame->decoded); + frame->ok = true; + txn_spill_file_counted_pad_0(frame); +} + +/* txn_spill_file_counted reads a storage's listed spill file back and counts its versions as they count at its transaction's entry. */ +static bool +txn_spill_file_counted(HostIoIo *io, HostSharedRecord state, const HostStorage *storage, const TxnSpillFile *file, + uint64 *result, HostError *error) +{ + txn_spill_file_counted_frame_0_t frame = {.io = io, .state = state, .storage = storage, .file = file, + .result = result, .error = error}; + + std_landing(&frame, txn_spill_file_counted_body_0, txn_spill_file_counted_pad_0); + return frame.ok; +} + +/* + * txn_spill_uncounted takes back at a spill file's transaction's entry what + * its rows, entries, and value and free space map versions added, once its + * removal from its manifest took it out, read before any checkpoint after + * that removal unlinks it. + */ +static bool +txn_spill_uncounted(HostIoIo *io, HostSharedRecord state, const HostStorage *storage, const TxnSpillFile *file, + HostError *error) +{ + uint64 count; + HostXid xid = {file->xid}; + bool has; + uint64 added; + + if (!txn_spill_entered(file->kind)) + return true; + return txn_spill_file_counted(io, state, storage, file, &count, error) && + txn_status_add(io, xid, std_wrapping_sub_u64(0, count), &has, &added, error); +} + +/* txn_spill_folded_frame_0_t is folded's state across its pad: the files listed, the xids applied, the counts, and the files removed, while live. */ +typedef struct txn_spill_folded_frame_0_t +{ + HostIoIo *io; + HostSharedRecord state; + HostLsn point; + HostError *error; + bool ok; + HostStorage storage; + txn_TupleVecFileU64_t listed; + bool listed_live; + /* applied is a Vec. */ + StdVec applied; + bool applied_live; + /* counts is a Vec<(File, u64)>. */ + StdVec counts; + bool counts_live; + /* gone is a Vec. */ + StdVec gone; + bool gone_live; +} txn_spill_folded_frame_0_t; + +/* txn_spill_folded_pad_0 is folded's pad: what it holds, while live. */ +static void +txn_spill_folded_pad_0(void *frame_) +{ + txn_spill_folded_frame_0_t *frame = frame_; + + if (frame->gone_live) + { + frame->gone_live = false; + std_vec_free_Checked(&frame->gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + } if (frame->counts_live) { frame->counts_live = false; @@ -3953,95 +5232,106 @@ txn_spill_folded_pad_0(void *frame_) } } -/* txn_spill_folded_counted reads a listed spill file back and counts its versions at its transaction's entry, into the frame's counts. */ +/* txn_spill_folded_frozen reports whether a spill file is a frozen insert's, of a writer below the first normal xid, that a fold at a point passes. */ static bool -txn_spill_folded_counted(txn_spill_folded_frame_0_t *frame, const TxnSpillFile *file) +txn_spill_folded_frozen(const TxnSpillFile *file, HostLsn point) { - TxnRunsNames names; - ChunksErrorError failed; - txn_TupleFileU64_t count = {*file}; + return file->xid < TXN_STATUS_FIRST_NORMAL && file->passed <= point; +} - if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, frame->state, &frame->storage, file->kind, file->xid, &names, - frame->error)) - return false; - txn_runs_spill_directory(frame->storage.database, txn_state_own(frame->state), &frame->path); - frame->path_live = true; - txn_runs_spill_name(&names, file->number, &frame->name); - frame->name_live = true; +/* txn_spill_folded_freed reports whether the listed files hold a free space map's spill of an xid. */ +static bool +txn_spill_folded_freed(const StdVec *listed, uint64 xid) +{ + for (size_t k = 0; k < listed->len; k++) { - StdPathBuf joined; + const TxnSpillFile *file = &STD_VEC_AT(listed, TxnSpillFile, k); - std_path_join(frame->path.bytes.ptr, frame->path.bytes.len, (const char *) frame->name.ptr, frame->name.len, - &joined); - std_vec_free_Checked(&frame->path.bytes, 1, 1); - frame->path = joined; - } - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - if (!chunks_file_read_file(HOST_IO_IO_READ_DOORS(frame->io), frame->path.bytes.ptr, frame->path.bytes.len, - &frame->bytes, &failed)) - { - *frame->error = txn_base_storage(&failed); - return false; - } - frame->bytes_live = true; - if (!std_vec_try_reserve(&frame->counts, 1, sizeof(count), _Alignof(txn_TupleFileU64_t))) - { - *frame->error = host_Error_out_of_memory(); - return false; + if (file->xid == xid && file->kind == TXN_RUNS_KIND_SPILLED_FREE) + return true; } - if (!txn_runs_decode(frame->bytes.ptr, frame->bytes.len, &names, file->hash, &frame->decoded, frame->error)) - return false; - frame->decoded_live = true; - count.f1 = txn_spill_counted(&frame->decoded); - frame->decoded_live = false; - txn_runs_Decoded_drop(&frame->decoded); - frame->bytes_live = false; - std_vec_free_Checked(&frame->bytes, 1, 1); - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - std_vec_push_Checked(&frame->counts, &count, sizeof(count), _Alignof(txn_TupleFileU64_t)); - return true; + return false; } -/* txn_spill_folded_gather gathers the xids whose commits the fold applies and the counts of their files. */ +/* + * txn_spill_folded_gather gathers the xids whose versions the fold applies, + * by their commits or, for an xid with a free space map's spill listed, its + * abort, deciding every xid before it counts a file, so a transaction a crash + * ended, whose other spills stay listed until a checkpoint names them, is + * counted whatever its files' order; then it gathers the counts of their + * files, but the residuals the fold at the point listed. + */ static bool txn_spill_folded_gather(txn_spill_folded_frame_0_t *frame) { for (size_t k = 0; k < frame->listed.f0.len; k++) { const TxnSpillFile *file = &STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k); - bool known = txn_spill_has_xid(&frame->applied, file->xid); + HostXid xid = {file->xid}; + TxnStatusCommit commit; + bool applies; - if (!known) + if (file->xid < TXN_STATUS_FIRST_NORMAL || txn_spill_has_xid(&frame->applied, file->xid)) + continue; + if (!txn_status_committed(frame->io, xid, &commit, frame->error)) + return false; + if (commit.tag == TXN_STATUS_COMMIT_AT) + applies = commit.u.at <= frame->point; + else { - HostXid xid = {file->xid}; - TxnStatusCommit commit; + HostStatus status = HOST_STATUS_IN_PROGRESS; - if (!txn_status_committed(frame->io, xid, &commit, frame->error)) + if (txn_spill_folded_freed(&frame->listed.f0, file->xid) && + !host_status(HOST_IO_IO_IO_DOORS(frame->io), xid, &status, frame->error)) return false; - if (commit.tag != TXN_STATUS_COMMIT_AT || commit.u.at > frame->point) - continue; - if (!std_vec_try_reserve(&frame->applied, 1, sizeof(uint64), _Alignof(uint64))) - { - *frame->error = host_Error_out_of_memory(); - return false; - } - std_vec_push_Checked(&frame->applied, &file->xid, sizeof(uint64), _Alignof(uint64)); + applies = status == HOST_STATUS_ABORTED; + } + if (!applies) + continue; + if (!std_vec_try_reserve(&frame->applied, 1, sizeof(uint64), _Alignof(uint64))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(&frame->applied, &file->xid, sizeof(uint64), _Alignof(uint64)); + } + for (size_t k = 0; k < frame->listed.f0.len; k++) + { + const TxnSpillFile *file = &STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k); + txn_TupleFileU64_t count = {*file}; + + if (file->xid < TXN_STATUS_FIRST_NORMAL || !txn_spill_has_xid(&frame->applied, file->xid) || + !txn_spill_entered(file->kind) || file->from == frame->point) + continue; + if (!std_vec_try_reserve(&frame->counts, 1, sizeof(count), _Alignof(txn_TupleFileU64_t))) + { + *frame->error = host_Error_out_of_memory(); + return false; } - if ((file->kind == TXN_RUNS_KIND_SPILLED_ROWS || file->kind == TXN_RUNS_KIND_SPILLED_ENTRIES || - file->kind == TXN_RUNS_KIND_SPILLED_VALUES) && !txn_spill_folded_counted(frame, file)) + if (!txn_spill_file_counted(frame->io, frame->state, &frame->storage, file, &count.f1, frame->error)) return false; + std_vec_push_Checked(&frame->counts, &count, sizeof(count), _Alignof(txn_TupleFileU64_t)); } return true; } +/* txn_spill_folded_closure_0 is folded's removal's test: a frozen file the point passes, or an applied transaction's file that is no residual of the fold. */ +static bool +txn_spill_folded_closure_0(void *captures, const TxnSpillFile *file) +{ + const txn_spill_folded_frame_0_t *frame = captures; + + return txn_spill_folded_frozen(file, frame->point) || + (txn_spill_has_xid(&frame->applied, file->xid) && file->from != frame->point); +} + /* txn_spill_folded_body_0 is folded's body. */ static void txn_spill_folded_body_0(void *frame_) { txn_spill_folded_frame_0_t *frame = frame_; TxnStatePersistence persistence; + bool frozen = false; if (!txn_state_described(frame->state, &frame->storage, &persistence)) { @@ -4061,7 +5351,9 @@ txn_spill_folded_body_0(void *frame_) txn_spill_folded_pad_0(frame); return; } - if (frame->applied.len == 0) + for (size_t k = 0; k < frame->listed.f0.len && !frozen; k++) + frozen = txn_spill_folded_frozen(&STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k), frame->point); + if (frame->applied.len == 0 && !frozen) { frame->ok = true; txn_spill_folded_pad_0(frame); @@ -4069,7 +5361,7 @@ txn_spill_folded_body_0(void *frame_) } if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-spill-folded-counted", frame->error) || !txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->state, &frame->storage, - txn_spill_xids_closure_0, &frame->applied, &frame->gone, frame->error)) + txn_spill_folded_closure_0, frame, &frame->gone, frame->error)) { txn_spill_folded_pad_0(frame); return; @@ -4102,9 +5394,16 @@ txn_spill_folded_body_0(void *frame_) /* * txn_spill_folded takes out of a storage's manifest the spills of every * transaction whose commit a fold at a point applies, and takes back at - * each transaction's entry what its spilled rows, entries, and value map - * versions added, reading each file's count before it leaves the manifest; - * only the counts of the files its own removal took out go back. + * each transaction's entry what its spilled rows, entries, and value and + * free space map versions added, reading each file's count before it leaves + * the manifest; only the counts of the files its own removal took out go + * back. An aborted transaction's free space map spill, which its abort + * left, goes too, the fold having applied its takes as it applies an aborted + * take in the memtable, and so does a file of a writer below the first + * normal xid, a frozen insert's, which counts nothing, once the point + * reaches its passing point. A residual the fold at the point listed stays, + * holding the versions it kept, and takes the place of the rest of its + * transaction's files of its kind (bugs row 228). */ bool txn_spill_folded(HostIoIo *io, HostSharedRecord state, HostLsn point, HostError *error) @@ -4115,863 +5414,873 @@ txn_spill_folded(HostIoIo *io, HostSharedRecord state, HostLsn point, HostError return frame.ok; } -/* txn_spill_unfolded_frame_0_t is unfolded_since's state across its pad: the newest base, the hold, and the next key found, while live. */ -typedef struct txn_spill_unfolded_frame_0_t +/* txn_TupleRefSliceU8RefSliceU8U_t is a residual's kept version, (&[u8], &[u8], usize): its key, its record's bytes, and its place among its group's. */ +typedef struct txn_TupleRefSliceU8RefSliceU8U_t { - HostIoIo *io; - const HostSharedHeld *held; - HostLsn end; - bool *result; + StdBytes f0; + StdBytes f1; + size_t f2; +} txn_TupleRefSliceU8RefSliceU8U_t; + +/* TxnSpillGroup is Group: a residual's transaction and kind, its spill's names, and the kept versions it holds. */ +typedef struct TxnSpillGroup +{ + uint64 xid; + TxnRunsKind kind; + TxnRunsNames names; + /* versions is a Vec<(&[u8], &[u8], usize)>. */ + StdVec versions; +} TxnSpillGroup; + +/* txn_spill_kept_cmp orders a group's kept versions by their keys, then their places, as a stable sort by key does. */ +static int +txn_spill_kept_cmp(const void *a, const void *b) +{ + const txn_TupleRefSliceU8RefSliceU8U_t *x = a; + const txn_TupleRefSliceU8RefSliceU8U_t *y = b; + int order = (int) std_bytes_Ord_cmp(x->f0.ptr, x->f0.len, y->f0.ptr, y->f0.len); + + if (order != 0) + return order; + return x->f2 < y->f2 ? -1 : x->f2 > y->f2; +} + +/* txn_spill_group_written_frame_0_t is group_written's state across its pad: the writing, the distinct keys, the filter, and a key's records, while live. */ +typedef struct txn_spill_group_written_frame_0_t +{ + HostIoIoDoors io; + const StdPathBuf *dir; + const TxnSpillGroup *group; + uint32 number; + HostLsn point; + TxnSpillFile *result; HostError *error; bool ok; - TxnBaseBase base; - bool base_live; - HostDeltaHold hold; - bool hold_live; - txn_TupleVecU8VecVersion_t next; - bool next_live; -} txn_spill_unfolded_frame_0_t; + TxnRunsWriting *writing; + bool writing_live; + /* keys is a Vec<&[u8]> of the distinct keys. */ + StdVec keys; + bool keys_live; + TxnSpillFilter filter; + bool filter_live; + /* records is a Vec<&[u8]> of a key's records. */ + StdVec records; + bool records_live; +} txn_spill_group_written_frame_0_t; -/* txn_spill_unfolded_pad_0 is unfolded_since's pad: what it holds, while live. */ +/* txn_spill_group_written_pad_0 is group_written's pad: what it holds, while live. */ static void -txn_spill_unfolded_pad_0(void *frame_) +txn_spill_group_written_pad_0(void *frame_) { - txn_spill_unfolded_frame_0_t *frame = frame_; + txn_spill_group_written_frame_0_t *frame = frame_; - if (frame->next_live) + if (frame->records_live) { - frame->next_live = false; - std_vec_free_Checked(&frame->next.f0, 1, 1); - txn_memtable_version_versions_drop(&frame->next.f1, 0); + frame->records_live = false; + std_vec_free_Checked(&frame->records, sizeof(StdBytes), _Alignof(StdBytes)); } - if (frame->hold_live) + if (frame->filter_live) { - frame->hold_live = false; - host_delta_Hold_drop(&frame->hold); + frame->filter_live = false; + std_vec_free_Checked(&frame->filter.words, sizeof(uint64), _Alignof(uint64)); } - if (frame->base_live) + if (frame->keys_live) { - frame->base_live = false; - txn_base_Base_drop(&frame->base); + frame->keys_live = false; + std_vec_free_Checked(&frame->keys, sizeof(StdBytes), _Alignof(StdBytes)); + } + if (frame->writing_live) + { + frame->writing_live = false; + txn_runs_Writing_drop(frame->writing); } } -/* txn_spill_unfolded_body_0 is unfolded_since's body. */ +/* txn_spill_group_written_body_0 is group_written's body. */ static void -txn_spill_unfolded_body_0(void *frame_) +txn_spill_group_written_body_0(void *frame_) { - txn_spill_unfolded_frame_0_t *frame = frame_; - HostSharedRecord record = host_shared_Held_record(frame->held); - HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + txn_spill_group_written_frame_0_t *frame = frame_; + const StdVec *versions = &frame->group->versions; + TxnRunsTail none = {.tag = TXN_RUNS_TAIL_NONE}; bool some; - uint64 word; + RepoRunFile run; - *frame->result = false; - if (!txn_state_newest(frame->io, frame->held, &some, &frame->base, frame->error)) - return; - frame->base_live = some; - if (some && frame->base.folded >= frame->end) + std_vec_new(&frame->keys, sizeof(StdBytes), _Alignof(StdBytes)); + frame->keys_live = true; + std_vec_new(&frame->records, sizeof(StdBytes), _Alignof(StdBytes)); + frame->records_live = true; + if (!std_vec_try_reserve_exact(&frame->keys, versions->len, sizeof(StdBytes), _Alignof(StdBytes)) || + !std_vec_try_reserve_exact(&frame->records, versions->len, sizeof(StdBytes), _Alignof(StdBytes))) { - frame->ok = true; - txn_spill_unfolded_pad_0(frame); + *frame->error = host_Error_out_of_memory(); + txn_spill_group_written_pad_0(frame); return; } - if (host_shared_Record_atomic(record, TXN_STATE_SPILLS_INDEX, load) != 0 || - host_shared_Record_atomic(record, TXN_STATE_SPILLS_PENDING_INDEX, load) != 0 || - txn_memtable_table_memtable(&TXN_MEMTABLE_MAP_VALUES_TREE_OPS, record, &word)) + for (size_t k = 0; k < versions->len; k++) { - *frame->result = true; - frame->ok = true; - txn_spill_unfolded_pad_0(frame); + StdBytes key = STD_VEC_AT(versions, txn_TupleRefSliceU8RefSliceU8U_t, k).f0; + const StdBytes *last = frame->keys.len == 0 ? NULL : + &STD_VEC_AT(&frame->keys, StdBytes, frame->keys.len - 1); + + if (last == NULL || last->len != key.len || memcmp(last->ptr, key.ptr, key.len) != 0) + std_vec_push_Checked(&frame->keys, &key, sizeof(key), _Alignof(StdBytes)); + } + if (!txn_spill_Filter_of(frame->keys.len, frame->keys.ptr, frame->keys.len, &frame->filter, frame->error)) + { + txn_spill_group_written_pad_0(frame); return; } - if (!host_delta_hold(frame->io, &frame->hold, frame->error)) + frame->filter_live = true; + frame->writing = txn_runs_Writing_spill(frame->dir->bytes.ptr, frame->dir->bytes.len, &frame->group->names, none, + frame->number); + frame->writing_live = true; + for (size_t k = 0, at = 0; k < frame->keys.len; k++) { - txn_spill_unfolded_pad_0(frame); + const StdBytes *key = &STD_VEC_AT(&frame->keys, StdBytes, k); + + frame->records.len = 0; + for (; at < versions->len; at++) + { + const txn_TupleRefSliceU8RefSliceU8U_t *version = &STD_VEC_AT(versions, txn_TupleRefSliceU8RefSliceU8U_t, + at); + + if (version->f0.len != key->len || memcmp(version->f0.ptr, key->ptr, key->len) != 0) + break; + std_vec_push_Checked(&frame->records, &version->f1, sizeof(StdBytes), _Alignof(StdBytes)); + } + if (!txn_runs_Writing_key(frame->writing, frame->io, key->ptr, key->len, frame->records.ptr, + frame->records.len, frame->error)) + { + txn_spill_group_written_pad_0(frame); + return; + } + } + if (!txn_runs_Writing_end(frame->writing, frame->io, frame->filter.words.ptr, frame->filter.words.len, &some, + &run, frame->error)) + { + txn_spill_group_written_pad_0(frame); return; } - frame->hold_live = true; - if (!txn_memtable_table_next(frame->io, &frame->hold, record, false, NULL, 0, TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, - &some, &frame->next, frame->error)) + if (!some) { - txn_spill_unfolded_pad_0(frame); + *frame->error = txn_spill_internal("a residual wrote no key"); + txn_spill_group_written_pad_0(frame); return; } - frame->next_live = some; - *frame->result = some; + *frame->result = (TxnSpillFile) + { + .xid = frame->group->xid, .kind = frame->group->kind, .number = frame->number, .len = run.len, + .hash = run.hash, .bits = txn_spill_Filter_bits(&frame->filter), .from = frame->point + }; frame->ok = true; - txn_spill_unfolded_pad_0(frame); + txn_spill_group_written_pad_0(frame); } +/* txn_spill_group_written writes a residual's file of a group's kept versions, sorted by key, each key's newest first, with its filter, synced. */ +static bool +txn_spill_group_written(HostIoIoDoors io, const StdPathBuf *dir, const TxnSpillGroup *group, uint32 number, + HostLsn point, TxnSpillFile *result, HostError *error) +{ + txn_spill_group_written_frame_0_t frame = {.io = io, .dir = dir, .group = group, .number = number, + .point = point, .result = result, .error = error}; + + std_landing(&frame, txn_spill_group_written_body_0, txn_spill_group_written_pad_0); + return frame.ok; +} + +/* txn_spill_residual_captures_t is residual's edit's captures: the state, the groups, the slots' spills, the directory, the point, and the files it made. */ +typedef struct txn_spill_residual_captures_t +{ + HostSharedRecord state; + const StdVec *groups; + const StdVec *slotted; + const StdPathBuf *dir; + HostLsn point; + StdVec *made; +} txn_spill_residual_captures_t; + /* - * txn_spill_unfolded_since reports whether a storage state's newest base is - * folded below a commit's end while the state holds a row, entry, or value - * map version that no fold applied, in its memtables or its spills. + * txn_spill_residual_closure_0 is residual's edit: each group's file written, + * numbered past its transaction's newest file of the kind listed or pended in + * a slot through the state's numbered, and listed after that newest listed + * file. */ static bool -txn_spill_unfolded_since(HostIoIo *io, const HostSharedHeld *held, HostLsn end, bool *result, HostError *error) +txn_spill_residual_closure_0(void *captures, HostSharedLongIo *io, StdVec *files, HostError *error) { - txn_spill_unfolded_frame_0_t frame = {.io = io, .held = held, .end = end, .result = result, .error = error}; + const txn_spill_residual_captures_t *c = captures; - std_landing(&frame, txn_spill_unfolded_body_0, txn_spill_unfolded_pad_0); - return frame.ok; + if (!std_vec_try_reserve_exact(c->made, c->groups->len, sizeof(TxnSpillFile), _Alignof(TxnSpillFile))) + { + *error = host_Error_out_of_memory(); + return false; + } + for (size_t g = 0; g < c->groups->len; g++) + { + const TxnSpillGroup *group = &STD_VEC_AT(c->groups, TxnSpillGroup, g); + uint32 past = 0; + size_t at = files->len; + TxnSpillFile file; + + for (size_t k = 0; k < files->len; k++) + { + const TxnSpillFile *listed = &STD_VEC_AT(files, TxnSpillFile, k); + + if (listed->xid == group->xid && listed->kind == group->kind) + { + past = Max(past, listed->number + 1); + at = k + 1; + } + } + for (size_t k = 0; k < c->slotted->len; k++) + { + const txn_TupleU64KindU32OptionU32_t *slot = &STD_VEC_AT(c->slotted, txn_TupleU64KindU32OptionU32_t, k); + + if (slot->f0 == group->xid && slot->f1 == group->kind) + past = Max(past, slot->f2 + 1); + } + if (!txn_spill_group_written(HOST_SHARED_LONG_IO_IO_DOORS(io), c->dir, group, + txn_state_numbered(c->state, past), c->point, &file, error)) + return false; + std_vec_insert(files, at, &file, sizeof(file), _Alignof(TxnSpillFile), RUST_FILE, 1418, 13); + std_vec_push_Checked(c->made, &file, sizeof(file), _Alignof(TxnSpillFile)); + } + return true; } -/* txn_spill_chunks_ended_frame_0_t is chunks_ended's state across its pad: the files listed, the xids seen and ended, the database's states, and the files removed, while live. */ -typedef struct txn_spill_chunks_ended_frame_0_t +/* txn_spill_residual_frame_0_t is residual's state across its pad: the groups, the slots' spills, and the directory, while live. */ +typedef struct txn_spill_residual_frame_0_t { HostIoIo *io; - HostSharedRecord deltas; + HostSharedRecord state; + HostLsn point; + const TxnSpillKept *kept; + size_t kept_len; + StdVec *result; HostError *error; bool ok; HostStorage storage; - txn_TupleVecFileU64_t listed; - bool listed_live; - /* seen and ended are Vecs of xids. */ - StdVec seen; - bool seen_live; - StdVec ended; - bool ended_live; - /* states is a Vec<(u64, Held)>. */ - StdVec states; - bool states_live; - StdVec gone; - bool gone_live; -} txn_spill_chunks_ended_frame_0_t; + /* groups is a Vec. */ + StdVec groups; + bool groups_live; + /* slotted is a Vec. */ + StdVec slotted; + bool slotted_live; + StdPathBuf dir; + bool dir_live; -/* txn_spill_states_drop drops a Vec<(u64, Held)>. */ -static void -txn_spill_states_drop(StdVec *states) -{ - for (size_t k = 0; k < states->len; k++) - host_shared_Held_drop(&STD_VEC_AT(states, txn_TupleU64HeldState_t, k).f1); - std_vec_free_Checked(states, sizeof(txn_TupleU64HeldState_t), _Alignof(txn_TupleU64HeldState_t)); -} + /* + * counted is how many groups have their counts added at their entries + * with their files not yet listed, which the pad gives back. + */ + size_t counted; +} txn_spill_residual_frame_0_t; -/* txn_spill_chunks_ended_pad_0 is chunks_ended's pad: what it holds, while live. */ +static void txn_spill_residual_given_back(txn_spill_residual_frame_0_t *frame, size_t groups); + +/* txn_spill_residual_pad_0 is residual's pad: the counts of groups not listed, then what it holds, while live. */ static void -txn_spill_chunks_ended_pad_0(void *frame_) +txn_spill_residual_pad_0(void *frame_) { - txn_spill_chunks_ended_frame_0_t *frame = frame_; + txn_spill_residual_frame_0_t *frame = frame_; - if (frame->gone_live) - { - frame->gone_live = false; - std_vec_free_Checked(&frame->gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - } - if (frame->states_live) + if (frame->counted > 0) { - frame->states_live = false; - txn_spill_states_drop(&frame->states); + txn_spill_residual_given_back(frame, frame->counted); + frame->counted = 0; } - if (frame->ended_live) + if (frame->dir_live) { - frame->ended_live = false; - std_vec_free_Checked(&frame->ended, sizeof(uint64), _Alignof(uint64)); + frame->dir_live = false; + std_vec_free_Checked(&frame->dir.bytes, 1, 1); } - if (frame->seen_live) + if (frame->slotted_live) { - frame->seen_live = false; - std_vec_free_Checked(&frame->seen, sizeof(uint64), _Alignof(uint64)); + frame->slotted_live = false; + std_vec_free_Checked(&frame->slotted, sizeof(txn_TupleU64KindU32OptionU32_t), + _Alignof(txn_TupleU64KindU32OptionU32_t)); } - if (frame->listed_live) + if (frame->groups_live) { - frame->listed_live = false; - std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + frame->groups_live = false; + for (size_t k = 0; k < frame->groups.len; k++) + std_vec_free_Checked(&STD_VEC_AT(&frame->groups, TxnSpillGroup, k).versions, + sizeof(txn_TupleRefSliceU8RefSliceU8U_t), + _Alignof(txn_TupleRefSliceU8RefSliceU8U_t)); + std_vec_free_Checked(&frame->groups, sizeof(TxnSpillGroup), _Alignof(TxnSpillGroup)); } } -/* txn_spill_chunks_ended_gone reports whether a committed transaction's chunk spills may go: its status entry gone, or no state of the repository unfolded past its commit. */ +/* txn_spill_residual_grouped groups the kept versions by their writers' top-level xids and their kinds, each group with its spill's names. */ static bool -txn_spill_chunks_ended_gone(txn_spill_chunks_ended_frame_0_t *frame, HostXid xid, bool *gone) +txn_spill_residual_grouped(txn_spill_residual_frame_0_t *frame) { - TxnStatusCommit commit; - - if (!txn_status_committed(frame->io, xid, &commit, frame->error)) - return false; - *gone = commit.tag == TXN_STATUS_COMMIT_MISSING; - if (commit.tag != TXN_STATUS_COMMIT_AT) - return true; - if (!txn_state_states(frame->io, frame->storage.database, &frame->states, frame->error)) - return false; - frame->states_live = true; - *gone = true; - for (size_t k = 0; k < frame->states.len && *gone; k++) + std_vec_new(&frame->groups, sizeof(TxnSpillGroup), _Alignof(TxnSpillGroup)); + frame->groups_live = true; + for (size_t k = 0; k < frame->kept_len; k++) { - bool unfolded; + const TxnSpillKept *kept = &frame->kept[k]; + TxnMemtableVersionRecord record; + TxnMemtableVersionWriter writer; + uint64 xid; + TxnSpillGroup *group = NULL; + txn_TupleRefSliceU8RefSliceU8U_t version = {{kept->key.ptr, kept->key.len}, {kept->bytes.ptr, kept->bytes.len}}; - if (!txn_spill_unfolded_since(frame->io, &STD_VEC_AT(&frame->states, txn_TupleU64HeldState_t, k).f1, - commit.u.at, &unfolded, frame->error)) + if (!txn_memtable_version_stored(kept->bytes.ptr, kept->bytes.len, &record, frame->error)) return false; - *gone = !unfolded; + writer = txn_memtable_version_Record_writer(&record); + xid = writer.has_top ? writer.top : writer.xid; + for (size_t g = 0; g < frame->groups.len && group == NULL; g++) + { + TxnSpillGroup *at = &STD_VEC_AT(&frame->groups, TxnSpillGroup, g); + + if (at->xid == xid && at->kind == kept->kind) + group = at; + } + if (group == NULL) + { + TxnSpillGroup made = {.xid = xid, .kind = kept->kind}; + + if (!std_vec_try_reserve(&frame->groups, 1, sizeof(made), _Alignof(TxnSpillGroup))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, frame->state, &frame->storage, kept->kind, xid, + &made.names, frame->error)) + return false; + std_vec_new(&made.versions, sizeof(version), _Alignof(txn_TupleRefSliceU8RefSliceU8U_t)); + std_vec_push_Checked(&frame->groups, &made, sizeof(made), _Alignof(TxnSpillGroup)); + group = &STD_VEC_AT(&frame->groups, TxnSpillGroup, frame->groups.len - 1); + } + version.f2 = group->versions.len; + if (!std_vec_try_reserve(&group->versions, 1, sizeof(version), _Alignof(txn_TupleRefSliceU8RefSliceU8U_t))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(&group->versions, &version, sizeof(version), _Alignof(txn_TupleRefSliceU8RefSliceU8U_t)); + } + for (size_t g = 0; g < frame->groups.len; g++) + { + StdVec *versions = &STD_VEC_AT(&frame->groups, TxnSpillGroup, g).versions; + + qsort(versions->ptr, versions->len, sizeof(txn_TupleRefSliceU8RefSliceU8U_t), txn_spill_kept_cmp); } - frame->states_live = false; - txn_spill_states_drop(&frame->states); return true; } -/* txn_spill_chunks_ended_body_0 is chunks_ended's body. */ +/* + * txn_spill_residual_given_back takes the counts of the first groups back + * from their transactions' entries after a failed listing or a panic; a give-back that + * fails leaves an overcount, which only keeps an entry longer, where an + * undercount could free it under its listed versions. + */ static void -txn_spill_chunks_ended_body_0(void *frame_) +txn_spill_residual_given_back(txn_spill_residual_frame_0_t *frame, size_t groups) { - txn_spill_chunks_ended_frame_0_t *frame = frame_; + for (size_t k = 0; k < groups; k++) + { + const TxnSpillGroup *group = &STD_VEC_AT(&frame->groups, TxnSpillGroup, k); + HostXid xid = {group->xid}; + HostError error; + bool has; + uint64 added; + + if (!txn_status_add(frame->io, xid, std_wrapping_sub_u64(0, (uint64) group->versions.len), &has, &added, + &error)) + host_Error_drop(&error); + } +} + +/* txn_spill_residual_body_0 is residual's body. */ +static void +txn_spill_residual_body_0(void *frame_) +{ + txn_spill_residual_frame_0_t *frame = frame_; TxnStatePersistence persistence; + txn_spill_residual_captures_t captures; - if (!txn_spill_BranchDeltas_Home_described(frame->deltas, &frame->storage, &persistence)) + std_vec_new(frame->result, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (frame->kept_len == 0 || !txn_state_described(frame->state, &frame->storage, &persistence)) { frame->ok = true; return; } - if (!txn_spill_listed(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, frame->io, frame->deltas, &frame->storage, - &frame->listed, frame->error)) + if (!txn_spill_residual_grouped(frame)) + { + txn_spill_residual_pad_0(frame); return; - frame->listed_live = true; - std_vec_new(&frame->seen, sizeof(uint64), _Alignof(uint64)); - frame->seen_live = true; - std_vec_new(&frame->ended, sizeof(uint64), _Alignof(uint64)); - frame->ended_live = true; - for (size_t k = 0; k < frame->listed.f0.len; k++) + } + txn_runs_spill_directory(frame->storage.database, txn_state_own(frame->state), &frame->dir); + frame->dir_live = true; + if (!txn_spill_pended(&TXN_SPILL_STATE_HOME_OPS, frame->state, &frame->storage, &frame->slotted, frame->error)) { - HostXid xid = {STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k).xid}; - HostStatus status; - bool gone = false; + txn_spill_residual_pad_0(frame); + return; + } + frame->slotted_live = true; + for (size_t k = 0; k < frame->groups.len; k++) + { + const TxnSpillGroup *group = &STD_VEC_AT(&frame->groups, TxnSpillGroup, k); + HostXid xid = {group->xid}; + bool has; + uint64 added; - if (txn_spill_has_xid(&frame->seen, xid.f0)) - continue; - if (!std_vec_try_reserve(&frame->seen, 1, sizeof(uint64), _Alignof(uint64))) - { - *frame->error = host_Error_out_of_memory(); - txn_spill_chunks_ended_pad_0(frame); - return; - } - std_vec_push_Checked(&frame->seen, &xid.f0, sizeof(uint64), _Alignof(uint64)); - if (!host_status(HOST_IO_IO_IO_DOORS(frame->io), xid, &status, frame->error) || - (status == HOST_STATUS_COMMITTED && !txn_spill_chunks_ended_gone(frame, xid, &gone))) + if (!txn_status_add(frame->io, xid, (uint64) group->versions.len, &has, &added, frame->error)) { - txn_spill_chunks_ended_pad_0(frame); + txn_spill_residual_pad_0(frame); return; } - if (status != HOST_STATUS_COMMITTED || !gone) - continue; - if (!std_vec_try_reserve(&frame->ended, 1, sizeof(uint64), _Alignof(uint64))) - { - *frame->error = host_Error_out_of_memory(); - txn_spill_chunks_ended_pad_0(frame); - return; - } - std_vec_push_Checked(&frame->ended, &xid.f0, sizeof(uint64), _Alignof(uint64)); + frame->counted = k + 1; } - if (frame->ended.len > 0) + captures = (txn_spill_residual_captures_t) { - if (!txn_spill_removed(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, frame->io, frame->deltas, &frame->storage, - txn_spill_xids_closure_0, &frame->ended, &frame->gone, frame->error)) - { - txn_spill_chunks_ended_pad_0(frame); - return; - } - frame->gone_live = true; + frame->state, &frame->groups, &frame->slotted, &frame->dir, frame->point, frame->result + }; + if (!txn_spill_changed(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->state, &frame->storage, + txn_spill_residual_closure_0, &captures, frame->error)) + { + txn_spill_residual_pad_0(frame); + return; } + frame->counted = 0; frame->ok = true; - txn_spill_chunks_ended_pad_0(frame); + txn_spill_residual_pad_0(frame); } /* - * txn_spill_chunks_ended takes out of a working state's chunk spills' - * manifest the files of every committed transaction whose status entry is - * gone, or past whose commit no storage state of the repository holds an - * unfolded version that may name one of its chunks; a later checkpoint - * unlinks the files it took out. + * txn_spill_residual lists, before a fold at a point installs, a residual of + * each transaction's spills of a map kind whose versions the fold keeps + * (bugs row 228), all in one hold of the manifest's writers' lock, so a + * concurrent fold lists its own after: a file holding the kept versions + * alone, each key's newest first, numbered past the transaction's newest + * file of the kind listed or pended in a slot through the state's numbered, + * as the committed arm numbers its files, written and synced, listed at the + * place of that newest listed file, its versions counted at the + * transaction's entry before the hold, since the status lock can't be taken + * inside it, and given back if the listing fails, so a listed residual is + * always counted. Readers read it in place of every other file of its + * transaction and kind once the base's fold point reaches the fold's + * (files), so they see the kept versions in its sources until the install + * and in it after, and never hold its versions in the delta's budget. A fold + * at the same point keeps the same versions of the same sources, so two such + * residuals hold the same versions. It answers the files it listed, which a + * fold whose install is refused takes back out (unlisted). */ bool -txn_spill_chunks_ended(HostIoIo *io, HostSharedRecord deltas, HostError *error) +txn_spill_residual(HostIoIo *io, HostSharedRecord state, HostLsn point, const TxnSpillKept *kept, size_t kept_len, + StdVec *result, HostError *error) { - txn_spill_chunks_ended_frame_0_t frame = {.io = io, .deltas = deltas, .error = error}; + txn_spill_residual_frame_0_t frame = {.io = io, .state = state, .point = point, .kept = kept, + .kept_len = kept_len, .result = result, .error = error}; - std_landing(&frame, txn_spill_chunks_ended_body_0, txn_spill_chunks_ended_pad_0); + std_landing(&frame, txn_spill_residual_body_0, txn_spill_residual_pad_0); + if (!frame.ok) + std_vec_free_Checked(result, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); return frame.ok; } -/* TxnSpillRead is Read: a backend's read of a spill file, its path, length, and names, its index frame's place and entries, and the chunk it read last, by its entry. */ -typedef struct TxnSpillRead -{ - StdPathBuf path; - uint64 len; - TxnRunsNames names; - uint64 index_at; - /* entries is an Index, a Vec<(Vec, u64)>. */ - StdVec entries; - bool has_chunk; - size_t chunk_at; - /* chunk is a Chunk, a Vec<(Vec, Vec>)>. */ - StdVec chunk; -} TxnSpillRead; - -/* txn_spill_Read_drop is Read's drop glue: its path, entries, and chunk. */ -static void -txn_spill_Read_drop(void *elem) -{ - TxnSpillRead *self = elem; - - if (self->has_chunk) - txn_spill_spilled_drop(&self->chunk, 0); - txn_runs_Index_drop(&self->entries); - std_vec_free_Checked(&self->path.bytes, 1, 1); -} - -/* TXN_SPILL_KEPT is KEPT, the most spill files a backend keeps its reads of. */ -#define TXN_SPILL_KEPT 8 - -/* txn_spill_VecRead_drop is Vec's drop glue. */ -static void -txn_spill_VecRead_drop(void *value) -{ - std_vec_drop_Checked(value, sizeof(TxnSpillRead), _Alignof(TxnSpillRead), txn_spill_Read_drop); -} - -/* taken at Vec and IoMutex>, which spill.c alone uses. */ -#define HOST_LOCK_SCOPE static -#define HOST_LOCK_DECLARE -#define HOST_LOCK_DEFINE -#define HOST_LOCK_TAKEN_TS VecRead -#define HOST_LOCK_TAKEN_T StdVec -#define HOST_LOCK_TAKEN_DEFAULT(place) std_vec_new((place), sizeof(TxnSpillRead), _Alignof(TxnSpillRead)) -#define HOST_LOCK_TAKEN_DROP(place) txn_spill_VecRead_drop(place) -#define HOST_LOCK_TAKEN_TYPE TXN_SPILL_READS_LOCK_TYPE -#define HOST_LOCK_IO_MUTEX -#define HOST_LOCK_TS VecRead -#define HOST_LOCK_T StdVec -#define HOST_LOCK_DEFAULT(place) std_vec_new((place), sizeof(TxnSpillRead), _Alignof(TxnSpillRead)) -#include "doltgres/host/lock_template.h" - -/* txn_spill_IoMutexVecRead_drop is IoMutex>' drop glue: its value's. */ -static void -txn_spill_IoMutexVecRead_drop(void *value) +/* txn_spill_unlisted_closure_0 is unlisted's removal's test: one of the fold's residuals. */ +static bool +txn_spill_unlisted_closure_0(void *captures, const TxnSpillFile *file) { - HostLockIoMutexVecRead *self = value; + const StdVec *residuals = captures; - txn_spill_VecRead_drop(&self->value); + for (size_t k = 0; k < residuals->len; k++) + if (txn_spill_File_PartialEq_eq(&STD_VEC_AT(residuals, TxnSpillFile, k), file)) + return true; + return false; } -/* TXN_SPILL_READS_TYPE is IoMutex>' size, Default, and drop. */ -static const StdDefaultType TXN_SPILL_READS_TYPE = {sizeof(HostLockIoMutexVecRead), _Alignof(HostLockIoMutexVecRead), -host_lock_IoMutexVecRead_Default_default, txn_spill_IoMutexVecRead_drop}; - -/* - * TXN_SPILL_READS is READS: the backend's reads of the spill files it read - * last, the latest first, so a scan's steps read each chunk once; its - * BackendLocal header and the process's value, which follows it. - */ -static struct -{ - HostLocalBackendLocal local; - HostLockIoMutexVecRead process; -} TXN_SPILL_READS = {HOST_LOCAL_BACKEND_LOCAL_NEW}; - -/* txn_spill_chunk_frame_0_t is Read's chunk's state across its pad: the chunk it replaces, the file, and the chunk read, while live. */ -typedef struct txn_spill_chunk_frame_0_t +/* txn_spill_unlisted_frame_t is unlisted's state across its pad: the files it took out, while live. */ +typedef struct txn_spill_unlisted_frame_t { - TxnSpillRead *self; HostIoIo *io; - size_t entry; + HostSharedRecord state; + const StdVec *residuals; + StdVec gone; + bool gone_live; HostError *error; bool ok; - StdVec old; - bool old_live; - HostFile file; - bool file_live; - StdVec read; - bool read_live; -} txn_spill_chunk_frame_0_t; - -/* txn_spill_chunk_pad_0 is Read's chunk's pad: what it holds, while live. */ -static void -txn_spill_chunk_pad_0(void *frame_) -{ - txn_spill_chunk_frame_0_t *frame = frame_; - - if (frame->read_live) - { - frame->read_live = false; - txn_spill_spilled_drop(&frame->read, 0); - } - if (frame->file_live) - { - frame->file_live = false; - host_File_drop(&frame->file); - } - if (frame->old_live) - { - frame->old_live = false; - txn_spill_spilled_drop(&frame->old, 0); - } -} +} txn_spill_unlisted_frame_t; -/* txn_spill_chunk_body_0 is Read's chunk's body. */ +/* txn_spill_unlisted_body takes the residuals out and takes back each one's count. */ static void -txn_spill_chunk_body_0(void *frame_) +txn_spill_unlisted_body(void *frame_) { - txn_spill_chunk_frame_0_t *frame = frame_; - TxnSpillRead *self = frame->self; - ChunksErrorError failed; - uint64 start; - uint64 end = self->index_at; - size_t next; + txn_spill_unlisted_frame_t *frame = frame_; + HostStorage storage; + TxnStatePersistence persistence; - if (self->has_chunk && self->chunk_at == frame->entry) + if (frame->residuals->len == 0 || !txn_state_described(frame->state, &storage, &persistence)) { frame->ok = true; return; } - frame->old = self->chunk; - frame->old_live = self->has_chunk; - self->has_chunk = false; - if (!chunks_file_open_rw(HOST_IO_IO_IO_DOORS(frame->io), self->path.bytes.ptr, self->path.bytes.len, false, - &frame->file, &failed)) - { - *frame->error = txn_base_storage(&failed); - txn_spill_chunk_pad_0(frame); - return; - } - frame->file_live = true; - std_index_check(frame->entry, self->entries.len, RUST_FILE, 1679, 29); - start = STD_VEC_AT(&self->entries, txn_TupleVecU8U64_t, frame->entry).f1; - next = STD_ADD(usize, frame->entry, 1, 1680, 44); - if (next < self->entries.len) - end = STD_VEC_AT(&self->entries, txn_TupleVecU8U64_t, next).f1; - if (!txn_runs_spill_chunk(frame->io, &frame->file, &self->names, start, end, &frame->read, frame->error)) - { - txn_spill_chunk_pad_0(frame); + if (!txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->state, &storage, txn_spill_unlisted_closure_0, + (void *) frame->residuals, &frame->gone, frame->error)) return; - } - frame->read_live = true; - frame->file_live = false; - host_File_drop(&frame->file); - if (frame->old_live) - { - frame->old_live = false; - txn_spill_spilled_drop(&frame->old, 0); - } - self->chunk = frame->read; - self->chunk_at = frame->entry; - self->has_chunk = true; - frame->read_live = false; + frame->gone_live = true; + for (size_t k = 0; k < frame->gone.len; k++) + if (!txn_spill_uncounted(frame->io, frame->state, &storage, &STD_VEC_AT(&frame->gone, TxnSpillFile, k), + frame->error)) + return; frame->ok = true; } -/* txn_spill_Read_chunk reads the keys of the chunk an index entry starts into the read, once. */ -static bool -txn_spill_Read_chunk(TxnSpillRead *self, HostIoIo *io, size_t entry, HostError *error) +/* txn_spill_unlisted_pad frees the files unlisted took out, while live. */ +static void +txn_spill_unlisted_pad(void *frame_) { - txn_spill_chunk_frame_0_t frame = {.self = self, .io = io, .entry = entry, .error = error}; + txn_spill_unlisted_frame_t *frame = frame_; - std_landing(&frame, txn_spill_chunk_body_0, txn_spill_chunk_pad_0); - return frame.ok; + if (frame->gone_live) + std_vec_free_Checked(&frame->gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + frame->gone_live = false; } -/* txn_spill_Read_entries_before returns how many of the read's index entries start at or below a key, or below it when strictly. */ -static size_t -txn_spill_Read_entries_before(const TxnSpillRead *self, const uint8 *key, size_t key_len, bool strictly) +/* + * txn_spill_unlisted takes a fold's residuals back out of its storage's + * manifest when its install is refused, with their counts; a fold whose + * install raised leaves them for the next fold's drop or the start, which + * no reader reads meanwhile. + */ +bool +txn_spill_unlisted(HostIoIo *io, HostSharedRecord state, const StdVec *residuals, HostError *error) { - size_t low = 0; - size_t high = self->entries.len; - - while (low < high) - { - size_t middle = low + (high - low) / 2; - const StdVec *first = &STD_VEC_AT(&self->entries, txn_TupleVecU8U64_t, middle).f0; - StdOrdering order = std_bytes_Ord_cmp(first->ptr, first->len, key, key_len); + txn_spill_unlisted_frame_t frame = {.io = io, .state = state, .residuals = residuals, .error = error}; - if (order == STD_ORDERING_LESS || (!strictly && order == STD_ORDERING_EQUAL)) - low = middle + 1; - else - high = middle; - } - return low; + std_landing(&frame, txn_spill_unlisted_body, txn_spill_unlisted_pad); + txn_spill_unlisted_pad(&frame); + return frame.ok; } -/* txn_spill_vec_u8_clone is Vec's Clone, into a place. */ -static void -txn_spill_vec_u8_clone(const void *elem, void *result) +/* txn_spill_Chain_drop frees a carry's chain: its key and its versions' records. */ +void +txn_spill_Chain_drop(TxnSpillChain *self) { - const StdVec *record = elem; - - std_slice_to_vec_Checked(result, record->ptr, record->len, 1, 1); + std_vec_free_Checked(&self->key, 1, 1); + for (size_t k = 0; k < self->versions.len; k++) + std_vec_free_Checked(&STD_VEC_AT(&self->versions, txn_TupleU64VecU8_t, k).f1, 1, 1); + std_vec_free_Checked(&self->versions, sizeof(txn_TupleU64VecU8_t), _Alignof(txn_TupleU64VecU8_t)); } -/* txn_spill_versions_clone clones a Vec> of records. */ -static void -txn_spill_versions_clone(const StdVec *records, StdVec *result) +/* TxnSpillLayerFile is one file of a carry's layer: its writer's top-level xid and tree, with its keys, each with its records newest first. */ +typedef struct TxnSpillLayerFile { - std_slice_to_vec_clone(result, records->ptr, records->len, sizeof(StdVec), _Alignof(StdVec), - txn_spill_vec_u8_clone, txn_spill_vec_u8_drop); -} + uint64 xid; + TxnWrittenOf of; + /* keys is a Vec<(Vec, Vec>)>. */ + StdVec keys; +} TxnSpillLayerFile; -/* txn_spill_clone_frame_0_t is Spilled's clone's state across its pad: the key cloned, while live. */ -typedef struct txn_spill_clone_frame_0_t +/* txn_spill_KeyRecords_drop frees a key with its records. */ +static void +txn_spill_KeyRecords_drop(txn_TupleVecU8VecVecU8_t *self) { - const txn_TupleVecU8VecVecU8_t *self; - txn_TupleVecU8VecVecU8_t *result; - bool key_live; -} txn_spill_clone_frame_0_t; + std_vec_free_Checked(&self->f0, 1, 1); + for (size_t k = 0; k < self->f1.len; k++) + std_vec_free_Checked(&STD_VEC_AT(&self->f1, StdVec, k), 1, 1); + std_vec_free_Checked(&self->f1, sizeof(StdVec), _Alignof(StdVec)); +} -/* txn_spill_clone_pad_0 is Spilled's clone's pad: the key cloned, while live. */ +/* txn_spill_layers_drop frees a carry's layers, a Vec>. */ static void -txn_spill_clone_pad_0(void *frame_) +txn_spill_layers_drop(StdVec *layers) { - txn_spill_clone_frame_0_t *frame = frame_; - - if (frame->key_live) + for (size_t d = 0; d < layers->len; d++) { - frame->key_live = false; - std_vec_free_Checked(&frame->result->f0, 1, 1); + StdVec *layer = &STD_VEC_AT(layers, StdVec, d); + + for (size_t f = 0; f < layer->len; f++) + { + TxnSpillLayerFile *file = &STD_VEC_AT(layer, TxnSpillLayerFile, f); + + for (size_t k = 0; k < file->keys.len; k++) + txn_spill_KeyRecords_drop(&STD_VEC_AT(&file->keys, txn_TupleVecU8VecVecU8_t, k)); + std_vec_free_Checked(&file->keys, sizeof(txn_TupleVecU8VecVecU8_t), _Alignof(txn_TupleVecU8VecVecU8_t)); + } + std_vec_free_Checked(layer, sizeof(TxnSpillLayerFile), _Alignof(TxnSpillLayerFile)); } + std_vec_free_Checked(layers, sizeof(StdVec), _Alignof(StdVec)); } -/* txn_spill_clone_body_0 is Spilled's clone's body: the key, then its records. */ -static void -txn_spill_clone_body_0(void *frame_) +/* txn_spill_key_records_cmp orders a layer file's keys by their bytes. */ +static int +txn_spill_key_records_cmp(const void *a, const void *b) { - txn_spill_clone_frame_0_t *frame = frame_; + const txn_TupleVecU8VecVecU8_t *x = a; + const txn_TupleVecU8VecVecU8_t *y = b; - std_slice_to_vec_Checked(&frame->result->f0, frame->self->f0.ptr, frame->self->f0.len, 1, 1); - frame->key_live = true; - txn_spill_versions_clone(&frame->self->f1, &frame->result->f1); - frame->key_live = false; -} - -/* txn_spill_Spilled_clone is Spilled's Clone: a key of a spill's file with its records copied, which a scan's step keeps. */ -static void -txn_spill_Spilled_clone(const txn_TupleVecU8VecVecU8_t *self, txn_TupleVecU8VecVecU8_t *result) -{ - txn_spill_clone_frame_0_t frame = {.self = self, .result = result}; - - std_landing(&frame, txn_spill_clone_body_0, txn_spill_clone_pad_0); + return (int) std_bytes_Ord_cmp(x->f0.ptr, x->f0.len, y->f0.ptr, y->f0.len); } -/* txn_spill_Read_looked_up returns a key's versions' records, newest first, or none for a key the file doesn't hold. */ +/* + * txn_spill_layered splits a carry's chains, which it takes, into layers of + * spill files (P2): each chain's versions, oldest first, into runs of one + * writer's, its first run in the first layer, its second in the second, and + * so on, so a layer's files hold a key of a tree once and listing the layers + * oldest first keeps every chain's order, as readers read a key's versions + * across files in their listing's order. A chain's key is copied into each + * layer it reaches, since each file owns its keys. + */ static bool -txn_spill_Read_looked_up(TxnSpillRead *self, HostIoIo *io, const uint8 *key, size_t key_len, StdVec *result, - HostError *error) +txn_spill_layered(StdVec *chains, StdVec *layers, HostError *error) { - size_t entry = txn_spill_Read_entries_before(self, key, key_len, false); - size_t low = 0; - size_t high; + bool ok = true; - std_vec_new(result, sizeof(StdVec), _Alignof(StdVec)); - if (entry == 0) - return true; - if (!txn_spill_Read_chunk(self, io, entry - 1, error)) - return false; - high = self->chunk.len; - while (low < high) + std_vec_new(layers, sizeof(StdVec), _Alignof(StdVec)); + for (size_t c = 0; c < chains->len && ok; c++) { - size_t middle = low + (high - low) / 2; - const txn_TupleVecU8VecVecU8_t *found = &STD_VEC_AT(&self->chunk, txn_TupleVecU8VecVecU8_t, middle); - StdOrdering order = std_bytes_Ord_cmp(found->f0.ptr, found->f0.len, key, key_len); + TxnSpillChain *chain = &STD_VEC_AT(chains, TxnSpillChain, c); + size_t depth = 0; - if (order == STD_ORDERING_EQUAL) + for (size_t start = 0; start < chain->versions.len && ok; depth++) { - txn_spill_versions_clone(&found->f1, result); - return true; - } - if (order == STD_ORDERING_LESS) - low = middle + 1; - else - high = middle; - } - return true; -} + uint64 xid = STD_VEC_AT(&chain->versions, txn_TupleU64VecU8_t, start).f0; + size_t end = start; + txn_TupleVecU8VecVecU8_t keyed; + StdVec *layer; + TxnSpillLayerFile *file = NULL; + + while (end < chain->versions.len && STD_VEC_AT(&chain->versions, txn_TupleU64VecU8_t, end).f0 == xid) + end++; + if (layers->len == depth) + { + StdVec fresh; -/* txn_spill_Read_next returns the file's first key past a place the way a scan goes, with its versions' records, or nothing past its last. */ -static bool -txn_spill_Read_next(TxnSpillRead *self, HostIoIo *io, bool has_after, const uint8 *after, size_t after_len, - TxnMemtableTableDirection way, bool *some, txn_TupleVecU8VecVecU8_t *result, HostError *error) -{ - size_t entries = self->entries.len; - size_t entry; + ok = std_vec_try_reserve(layers, 1, sizeof(StdVec), _Alignof(StdVec)); + if (!ok) + break; + std_vec_new(&fresh, sizeof(TxnSpillLayerFile), _Alignof(TxnSpillLayerFile)); + std_vec_push_Checked(layers, &fresh, sizeof(fresh), _Alignof(StdVec)); + } + layer = &STD_VEC_AT(layers, StdVec, depth); + for (size_t f = 0; f < layer->len && file == NULL; f++) + if (STD_VEC_AT(layer, TxnSpillLayerFile, f).xid == xid && + STD_VEC_AT(layer, TxnSpillLayerFile, f).of == chain->of) + file = &STD_VEC_AT(layer, TxnSpillLayerFile, f); + if (file == NULL) + { + TxnSpillLayerFile made = {.xid = xid, .of = chain->of}; + + ok = std_vec_try_reserve(layer, 1, sizeof(made), _Alignof(TxnSpillLayerFile)); + if (!ok) + break; + std_vec_new(&made.keys, sizeof(txn_TupleVecU8VecVecU8_t), _Alignof(txn_TupleVecU8VecVecU8_t)); + std_vec_push_Checked(layer, &made, sizeof(made), _Alignof(TxnSpillLayerFile)); + file = &STD_VEC_AT(layer, TxnSpillLayerFile, layer->len - 1); + } + ok = std_vec_try_reserve(&file->keys, 1, sizeof(keyed), _Alignof(txn_TupleVecU8VecVecU8_t)); + if (!ok) + break; + std_vec_clone(&keyed.f0, &chain->key, 1, 1); + std_vec_with_capacity_Checked(&keyed.f1, end - start, sizeof(StdVec), _Alignof(StdVec)); + for (size_t v = end; v > start; v--) + { + txn_TupleU64VecU8_t *version = &STD_VEC_AT(&chain->versions, txn_TupleU64VecU8_t, v - 1); - *some = false; - if (!has_after) - entry = way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD ? 0 : std_saturating_sub_usize(entries, 1); - else if (way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD) - entry = std_saturating_sub_usize(txn_spill_Read_entries_before(self, after, after_len, false), 1); - else + std_vec_push_Checked(&keyed.f1, &version->f1, sizeof(StdVec), _Alignof(StdVec)); + std_vec_new(&version->f1, 1, 1); + } + std_vec_push_Checked(&file->keys, &keyed, sizeof(keyed), _Alignof(txn_TupleVecU8VecVecU8_t)); + start = end; + } + } + for (size_t c = 0; c < chains->len; c++) + txn_spill_Chain_drop(&STD_VEC_AT(chains, TxnSpillChain, c)); + std_vec_free_Checked(chains, sizeof(TxnSpillChain), _Alignof(TxnSpillChain)); + if (!ok) { - entry = txn_spill_Read_entries_before(self, after, after_len, true); - if (entry == 0) - return true; - entry--; + *error = host_Error_out_of_memory(); + txn_spill_layers_drop(layers); + return false; } - while (entry < entries) + for (size_t d = 0; d < layers->len; d++) { - const txn_TupleVecU8VecVecU8_t *found = NULL; + StdVec *layer = &STD_VEC_AT(layers, StdVec, d); - if (!txn_spill_Read_chunk(self, io, entry, error)) - return false; - for (size_t k = 0; k < self->chunk.len && found == NULL; k++) + for (size_t f = 0; f < layer->len; f++) { - size_t at = way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD ? k : self->chunk.len - 1 - k; - const txn_TupleVecU8VecVecU8_t *key = &STD_VEC_AT(&self->chunk, txn_TupleVecU8VecVecU8_t, at); - StdOrdering order = has_after ? std_bytes_Ord_cmp(key->f0.ptr, key->f0.len, after, after_len) : - STD_ORDERING_EQUAL; + StdVec *keys = &STD_VEC_AT(layer, TxnSpillLayerFile, f).keys; - if (!has_after || (way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD ? order == STD_ORDERING_GREATER : - order == STD_ORDERING_LESS)) - found = key; - } - if (found != NULL) - { - txn_spill_Spilled_clone(found, result); - *some = true; - return true; + qsort(keys->ptr, keys->len, sizeof(txn_TupleVecU8VecVecU8_t), txn_spill_key_records_cmp); } - if (way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD) - entry = STD_ADD(usize, entry, 1, 1728, 39); - else if (entry == 0) - return true; - else - entry--; } return true; } -/* txn_spill_reading_frame_0_t is reading's state across its pad: the backend's reads and their guard, the read it runs, and the file it opens, while live. */ -typedef struct txn_spill_reading_frame_0_t +/* TxnSpillSlot is a carried writer's status entry's slot, resolved before the carry's hold: its top-level xid, and its slot when it has an entry. */ +typedef struct TxnSpillSlot +{ + uint64 xid; + bool has; + TxnStatusRoomed roomed; +} TxnSpillSlot; + +/* txn_spill_carried_frame_0_t is carried's state across its pad: the spills' directory, the manifest's path, its writers' lock, the slots, the layers, the files, a file's writing, filter, keys, and tuples, and the counts, while live. */ +typedef struct txn_spill_carried_frame_0_t { HostIoIo *io; - const StdPathBuf *path; - const TxnRunsNames *names; - bool has_len; - uint64 len; - bool (*with) (void *captures, HostIoIo *io, TxnSpillRead *read, HostError *error); + const HostSharedHeld *held; + bool (*logged) (void *captures, HostSharedLongIo *io, StdVec *chains, HostError *error); void *captures; HostError *error; bool ok; - HostLocalLocal reads; - bool reads_live; - HostLockLockedRwLockWriteGuard guard; - bool guard_live; - TxnSpillRead read; - bool read_live; - HostFile file; - bool file_live; -} txn_spill_reading_frame_0_t; + HostStorage storage; + TxnRunsKind kinds[3]; + TxnRunsTail tails[3]; + StdPathBuf dir; + bool dir_live; + StdPathBuf path; + bool path_live; + HostSharedLongGuard writing; + bool writing_live; + HostSharedLongIo long_io; + StdVec slotted; + bool slotted_live; + StdVec chains; + StdVec layers; + bool layers_live; + StdVec files; + bool files_live; + TxnRunsWriting *writer; + bool writer_live; + TxnSpillFilter filter; + bool filter_live; -/* txn_spill_reading_pad_0 is reading's pad: what it holds, while live. */ + /* + * keys is a Vec<&[u8]> of a file's keys, and records a Vec<&[u8]> of a + * key's records. + */ + StdVec keys; + bool keys_live; + StdVec records; + bool records_live; + /* tuples is a Vec<(u64, Vec)>. */ + StdVec tuples; + bool tuples_live; + + /* + * counts is a Vec<(u64, u64)> of each file's writer and its versions' + * count. + */ + StdVec counts; + bool counts_live; + + /* + * writers are the top-level xids the carry may spill under, and slots + * their entries' slots. + */ + const uint64 *writers; + size_t writers_len; + StdVec slots; + bool slots_live; + /* made_from is where the carry's files start in the files listed. */ + size_t made_from; + + /* + * ends are the ends of the carry's records, ascending, and made the files + * it listed. + */ + StdVec ends; + bool ends_live; + StdVec made; + bool made_live; + StdVec body; + bool body_live; + HostDeltaHold hold; + bool hold_live; + HostDelayCheckpoint window; + bool window_live; +} txn_spill_carried_frame_0_t; + +/* txn_spill_carried_file_drop frees what one file of carried's holds. */ static void -txn_spill_reading_pad_0(void *frame_) +txn_spill_carried_file_drop(txn_spill_carried_frame_0_t *frame) { - txn_spill_reading_frame_0_t *frame = frame_; - - if (frame->file_live) + if (frame->tuples_live) { - frame->file_live = false; - host_File_drop(&frame->file); + frame->tuples_live = false; + for (size_t k = 0; k < frame->tuples.len; k++) + std_vec_free_Checked(&STD_VEC_AT(&frame->tuples, txn_TupleU64VecU8_t, k).f1, 1, 1); + std_vec_free_Checked(&frame->tuples, sizeof(txn_TupleU64VecU8_t), _Alignof(txn_TupleU64VecU8_t)); } - if (frame->read_live) + if (frame->records_live) { - frame->read_live = false; - txn_spill_Read_drop(&frame->read); + frame->records_live = false; + std_vec_free_Checked(&frame->records, sizeof(StdBytes), _Alignof(StdBytes)); } - if (frame->guard_live) + if (frame->keys_live) { - frame->guard_live = false; - host_lock_LockedRwLockWriteGuard_drop(&frame->guard); + frame->keys_live = false; + std_vec_free_Checked(&frame->keys, sizeof(StdBytes), _Alignof(StdBytes)); } - if (frame->reads_live) + if (frame->filter_live) { - frame->reads_live = false; - host_local_Local_drop(&frame->reads, &TXN_SPILL_READS_TYPE); + frame->filter_live = false; + std_vec_free_Checked(&frame->filter.words, sizeof(uint64), _Alignof(uint64)); } -} - -/* txn_spill_reading_lock locks the backend's reads, answering them. */ -static StdVec * -txn_spill_reading_lock(txn_spill_reading_frame_0_t *frame) -{ - frame->guard = host_lock_IoMutex_lock((HostLockIoMutex *) host_local_Local_Deref_deref(&frame->reads, - &TXN_SPILL_READS_TYPE), - &TXN_SPILL_READS_LOCK_TYPE, &frame->io->held); - frame->guard_live = true; - return host_lock_LockedRwLockWriteGuard_DerefMut_deref_mut(&frame->guard); -} - -/* txn_spill_reading_opened opens the file the backend keeps no read of and reads its index into the frame's read. */ -static bool -txn_spill_reading_opened(txn_spill_reading_frame_0_t *frame) -{ - ChunksErrorError failed; - TxnSpillRead *read = &frame->read; - - if (!chunks_file_open_rw(HOST_IO_IO_IO_DOORS(frame->io), frame->path->bytes.ptr, frame->path->bytes.len, false, - &frame->file, &failed)) + if (frame->writer_live) { - *frame->error = txn_base_storage(&failed); - return false; + frame->writer_live = false; + txn_runs_Writing_drop(frame->writer); } - frame->file_live = true; - std_PathBuf_Clone_clone(frame->path, &read->path); - read->names = *frame->names; - read->has_chunk = false; - read->len = frame->len; - std_vec_new(&read->entries, sizeof(txn_TupleVecU8U64_t), _Alignof(txn_TupleVecU8U64_t)); - frame->read_live = true; - if ((!frame->has_len && !host_size(HOST_IO_IO_READ_DOORS(frame->io), &frame->file, &read->len, frame->error)) || - !txn_runs_spill_index(frame->io, &frame->file, read->len, &read->index_at, &read->entries, frame->error)) - return false; - frame->file_live = false; - host_File_drop(&frame->file); - return true; } -/* txn_spill_reading_body_0 is reading's body. */ +/* txn_spill_carried_pad_0 is carried's pad: what it holds, while live. */ static void -txn_spill_reading_body_0(void *frame_) +txn_spill_carried_pad_0(void *frame_) { - txn_spill_reading_frame_0_t *frame = frame_; - StdVec *reads; - size_t at; + txn_spill_carried_frame_0_t *frame = frame_; - frame->reads = host_local_BackendLocal_get(&TXN_SPILL_READS.local, &TXN_SPILL_READS_TYPE); - frame->reads_live = true; - reads = txn_spill_reading_lock(frame); - for (at = 0; at < reads->len; at++) + if (frame->window_live) { - const TxnSpillRead *kept = &STD_VEC_AT(reads, TxnSpillRead, at); - - if (std_path_PartialEq_eq(kept->path.bytes.ptr, kept->path.bytes.len, frame->path->bytes.ptr, - frame->path->bytes.len) && (!frame->has_len || kept->len == frame->len)) - break; + frame->window_live = false; + host_DelayCheckpoint_drop(&frame->window); } - if (at < reads->len) + if (frame->hold_live) { - std_vec_remove(reads, at, &frame->read, sizeof(frame->read), _Alignof(TxnSpillRead), RUST_FILE, - 1651, 29); - frame->read_live = true; + frame->hold_live = false; + host_delta_Hold_drop(&frame->hold); } - frame->guard_live = false; - host_lock_LockedRwLockWriteGuard_drop(&frame->guard); - if (!frame->read_live && !txn_spill_reading_opened(frame)) + if (frame->body_live) { - txn_spill_reading_pad_0(frame); - return; + frame->body_live = false; + std_vec_free_Checked(&frame->body, 1, 1); } - frame->ok = frame->with(frame->captures, frame->io, &frame->read, frame->error); - reads = txn_spill_reading_lock(frame); - std_vec_insert(reads, 0, &frame->read, sizeof(frame->read), _Alignof(TxnSpillRead), RUST_FILE, - 1667, 11); - frame->read_live = false; - std_vec_truncate(reads, TXN_SPILL_KEPT, sizeof(TxnSpillRead), txn_spill_Read_drop); - txn_spill_reading_pad_0(frame); -} - -/* - * txn_spill_reading runs a read of a spill's file of the length its - * manifest lists, or of its own for a pending file's, keeping the read first - * among the backend's and reading the index of a file it doesn't keep. - */ -static bool -txn_spill_reading(HostIoIo *io, const StdPathBuf *path, const TxnRunsNames *names, bool has_len, uint64 len, - bool (*with) (void *captures, HostIoIo *io, TxnSpillRead *read, HostError *error), void *captures, - HostError *error) -{ - txn_spill_reading_frame_0_t frame = {.io = io, .path = path, .names = names, .has_len = has_len, .len = len, - .with = with, .captures = captures, .error = error}; - - std_landing(&frame, txn_spill_reading_body_0, txn_spill_reading_pad_0); - return frame.ok; -} - -/* txn_spill_consistent reads a storage's slots and then lists its manifest, starting over while a change of the manifest moved its generation between the two, setting its caller's live flags of the two. */ -static bool -txn_spill_consistent(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, const HostStorage *storage, - StdVec *slotted, bool *slotted_live, txn_TupleVecFileU64_t *listed, bool *listed_live, - HostError *error) -{ - HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; - - for (;;) + if (frame->made_live) { - uint64 before = host_shared_Record_atomic(state, home->changed, load); - - if (!txn_spill_pended(home, state, storage, slotted, error)) - return false; - *slotted_live = true; - if (!host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-slots-read", error) || - !txn_spill_listed(home, io, state, storage, listed, error)) - return false; - *listed_live = true; - if (before == listed->f1) - return true; - *listed_live = false; - std_vec_free_Checked(&listed->f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - *slotted_live = false; - std_vec_free_Checked(slotted, sizeof(txn_TupleU64KindU32OptionU32_t), _Alignof(txn_TupleU64KindU32OptionU32_t)); + frame->made_live = false; + std_vec_free_Checked(&frame->made, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); } -} - -/* TxnSpillReading is Reading: a spill file a reader reads, its xid, its kind, its number, and its length when its manifest lists it. */ -typedef struct TxnSpillReading -{ - uint64 xid; - TxnRunsKind kind; - uint32 number; - bool has_len; - uint64 len; -} TxnSpillReading; - -/* txn_spill_files_frame_0_t is files' state across its pad: the slots, the listed files, the spills' directory, a file's path, and the readings, while live. */ -typedef struct txn_spill_files_frame_0_t -{ - const TxnSpillHomeOps *home; - HostIoIo *io; - HostSharedRecord state; - const HostStorage *storage; - TxnRunsKind kind; - bool has_key; - const uint8 *key; - size_t key_len; - StdVec *result; - HostError *error; - bool ok; - bool result_live; - StdVec slotted; - bool slotted_live; - txn_TupleVecFileU64_t listed; - bool listed_live; - StdPathBuf dir; - bool dir_live; - StdString name; - bool name_live; - StdPathBuf path; - bool path_live; -} txn_spill_files_frame_0_t; - -/* txn_spill_files_pad_0 is files' pad: what it holds, while live. */ -static void -txn_spill_files_pad_0(void *frame_) -{ - txn_spill_files_frame_0_t *frame = frame_; - - if (frame->path_live) + if (frame->ends_live) { - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); + frame->ends_live = false; + std_vec_free_Checked(&frame->ends, sizeof(uint64), _Alignof(uint64)); } - if (frame->name_live) + if (frame->slots_live) { - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); + frame->slots_live = false; + std_vec_free_Checked(&frame->slots, sizeof(TxnSpillSlot), _Alignof(TxnSpillSlot)); } - if (frame->dir_live) + txn_spill_carried_file_drop(frame); + if (frame->counts_live) { - frame->dir_live = false; - std_vec_free_Checked(&frame->dir.bytes, 1, 1); + frame->counts_live = false; + std_vec_free_Checked(&frame->counts, sizeof(std_TupleU64U64_t), _Alignof(std_TupleU64U64_t)); } - if (frame->listed_live) + if (frame->files_live) { - frame->listed_live = false; - std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + frame->files_live = false; + std_vec_free_Checked(&frame->files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + } + if (frame->layers_live) + { + frame->layers_live = false; + txn_spill_layers_drop(&frame->layers); } if (frame->slotted_live) { @@ -4979,4024 +6288,7119 @@ txn_spill_files_pad_0(void *frame_) std_vec_free_Checked(&frame->slotted, sizeof(txn_TupleU64KindU32OptionU32_t), _Alignof(txn_TupleU64KindU32OptionU32_t)); } - if (frame->result_live) + if (frame->writing_live) { - frame->result_live = false; - std_vec_free_Checked(frame->result, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); + frame->writing_live = false; + host_shared_LongGuard_drop(&frame->writing); + } + if (frame->path_live) + { + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + } + if (frame->dir_live) + { + frame->dir_live = false; + std_vec_free_Checked(&frame->dir.bytes, 1, 1); } } -/* txn_spill_files_push adds a reading to the frame's result. */ +/* txn_spill_carried_push adds an element to one of carried's vectors, refusing it when the vector can't grow. */ static bool -txn_spill_files_push(txn_spill_files_frame_0_t *frame, const TxnSpillReading *reading) +txn_spill_carried_push(txn_spill_carried_frame_0_t *frame, StdVec *vector, const void *elem, size_t size, + size_t align) { - if (!std_vec_try_reserve(frame->result, 1, sizeof(*reading), _Alignof(TxnSpillReading))) + if (!std_vec_try_reserve(vector, 1, size, align)) { *frame->error = host_Error_out_of_memory(); return false; } - std_vec_push_Checked(frame->result, reading, sizeof(*reading), _Alignof(TxnSpillReading)); + std_vec_push_Checked(vector, elem, size, align); return true; } -/* txn_spill_files_hit reports whether the frame's key hits a listed file's filter. */ +/* + * txn_spill_carried_file writes one file of a layer: its keys and records + * under its writer's xid, numbered through the state's numbered past the + * writer's files of the kind listed or pended, with a row file's spilled + * tuple ids, and lists it, counting its versions but its locks' and taking a + * frozen writer's passing point from its records' ends. + */ static bool -txn_spill_files_hit(txn_spill_files_frame_0_t *frame, const TxnSpillFile *file, bool *hit) +txn_spill_carried_file(txn_spill_carried_frame_0_t *frame, const TxnSpillLayerFile *layered) { + HostSharedRecord record = host_shared_Held_record(frame->held); + HostIoIoDoors doors = HOST_SHARED_LONG_IO_IO_DOORS(&frame->long_io); + size_t at = layered->of == TXN_WRITTEN_OF_ROWS ? 0 : layered->of == TXN_WRITTEN_OF_FREE ? 1 : 2; + TxnRunsKind kind = frame->kinds[at]; TxnRunsNames names; + uint32 past = 0; + uint64 count = 0; + uint64 passed = 0; + bool some; + RepoRunFile run; + TxnSpillFile file; + uint32 number; + std_TupleU64U64_t counted; - if (!txn_spill_names(frame->home, frame->state, frame->storage, file->kind, file->xid, &names, frame->error)) + if (layered->of != TXN_WRITTEN_OF_ROWS && layered->of != TXN_WRITTEN_OF_FREE && + layered->of != TXN_WRITTEN_OF_VALUES) + { + *frame->error = txn_spill_internal("a carry spills only a storage's rows and maps"); return false; - txn_runs_spill_name(&names, file->number, &frame->name); - frame->name_live = true; - std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, - &frame->path); - frame->path_live = true; - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - if (!txn_spill_hits(frame->io, &frame->path, file, frame->key, frame->key_len, hit, frame->error)) + } + if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, record, &frame->storage, kind, layered->xid, &names, + frame->error)) return false; - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - return true; -} - -/* txn_spill_files_body_0 is files' body. */ -static void -txn_spill_files_body_0(void *frame_) -{ - txn_spill_files_frame_0_t *frame = frame_; - - if (!txn_spill_consistent(frame->home, frame->io, frame->state, frame->storage, &frame->slotted, - &frame->slotted_live, &frame->listed, &frame->listed_live, frame->error)) + for (size_t k = 0; k < frame->files.len; k++) { - txn_spill_files_pad_0(frame); - return; + const TxnSpillFile *listed = &STD_VEC_AT(&frame->files, TxnSpillFile, k); + + if (listed->xid == layered->xid && listed->kind == kind) + past = Max(past, listed->number + 1); } - txn_runs_spill_directory(frame->storage->database, frame->home->own(frame->state), &frame->dir); - frame->dir_live = true; - std_vec_new(frame->result, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); - frame->result_live = true; for (size_t k = 0; k < frame->slotted.len; k++) { const txn_TupleU64KindU32OptionU32_t *slot = &STD_VEC_AT(&frame->slotted, txn_TupleU64KindU32OptionU32_t, k); - TxnSpillReading reading = {slot->f0, slot->f1, slot->f2}; - if (txn_spill_tree(slot->f1, frame->kind) && !txn_spill_files_push(frame, &reading)) - { - txn_spill_files_pad_0(frame); - return; - } + if (slot->f0 == layered->xid && slot->f1 == kind) + past = Max(past, slot->f2 + 1); } - for (size_t k = 0; k < frame->listed.f0.len; k++) + number = txn_state_numbered(record, past); + std_vec_with_capacity_Checked(&frame->keys, layered->keys.len, sizeof(StdBytes), _Alignof(StdBytes)); + frame->keys_live = true; + for (size_t k = 0; k < layered->keys.len; k++) { - const TxnSpillFile *file = &STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k); - TxnSpillReading reading = {file->xid, file->kind, file->number, true, file->len}; - bool seen = false; - bool hit = true; + const StdVec *key = &STD_VEC_AT(&layered->keys, txn_TupleVecU8VecVecU8_t, k).f0; + StdBytes bytes = {key->ptr, key->len}; - for (size_t j = 0; j < frame->result->len && !seen; j++) - { - const TxnSpillReading *found = &STD_VEC_AT(frame->result, TxnSpillReading, j); + std_vec_push_Checked(&frame->keys, &bytes, sizeof(bytes), _Alignof(StdBytes)); + } + if (!txn_spill_Filter_of(frame->keys.len, frame->keys.ptr, frame->keys.len, &frame->filter, frame->error)) + return false; + frame->filter_live = true; + frame->writer = txn_runs_Writing_spill(frame->dir.bytes.ptr, frame->dir.bytes.len, &names, frame->tails[at], + number); + frame->writer_live = true; + std_vec_new(&frame->records, sizeof(StdBytes), _Alignof(StdBytes)); + frame->records_live = true; + std_vec_new(&frame->tuples, sizeof(txn_TupleU64VecU8_t), _Alignof(txn_TupleU64VecU8_t)); + frame->tuples_live = true; + for (size_t k = 0; k < layered->keys.len; k++) + { + const txn_TupleVecU8VecVecU8_t *keyed = &STD_VEC_AT(&layered->keys, txn_TupleVecU8VecVecU8_t, k); - seen = found->xid == file->xid && found->number == file->number; + frame->records.len = 0; + if (!std_vec_try_reserve(&frame->records, keyed->f1.len, sizeof(StdBytes), _Alignof(StdBytes))) + { + *frame->error = host_Error_out_of_memory(); + return false; } - if (!txn_spill_tree(file->kind, frame->kind) || seen || - txn_spill_replaced_by(frame->slotted.ptr, frame->slotted.len, file)) - continue; - if ((frame->has_key && !txn_spill_files_hit(frame, file, &hit)) || - (hit && !txn_spill_files_push(frame, &reading))) + for (size_t r = 0; r < keyed->f1.len; r++) { - txn_spill_files_pad_0(frame); - return; + const StdVec *bytes = &STD_VEC_AT(&keyed->f1, StdVec, r); + StdBytes lent = {bytes->ptr, bytes->len}; + TxnMemtableVersionRecord version; + TxnMemtableVersionOpRef op; + txn_TupleU64VecU8_t tuple; + uint64 lsn; + + std_vec_push_Checked(&frame->records, &lent, sizeof(lent), _Alignof(StdBytes)); + if (!txn_memtable_version_stored(bytes->ptr, bytes->len, &version, frame->error)) + return false; + if (layered->xid < TXN_STATUS_FIRST_NORMAL) + passed = Max(passed, txn_memtable_version_Record_lsn(&version, &lsn) ? lsn : 1); + op = txn_memtable_version_Record_op(&version); + if (op.tag != TXN_MEMTABLE_VERSION_OP_REF_LOCK && op.tag != TXN_MEMTABLE_VERSION_OP_REF_LOCK_ROW) + count++; + if (kind == TXN_RUNS_KIND_SPILLED_ROWS && txn_spill_tid(bytes->ptr, bytes->len, &tuple.f0)) + { + std_vec_clone(&tuple.f1, &keyed->f0, 1, 1); + if (!txn_spill_carried_push(frame, &frame->tuples, &tuple, sizeof(tuple), + _Alignof(txn_TupleU64VecU8_t))) + { + std_vec_free_Checked(&tuple.f1, 1, 1); + return false; + } + } } + if (!txn_runs_Writing_key(frame->writer, doors, keyed->f0.ptr, keyed->f0.len, frame->records.ptr, + frame->records.len, frame->error)) + return false; } - frame->result_live = false; - frame->ok = true; - txn_spill_files_pad_0(frame); + if (!txn_runs_Writing_end(frame->writer, doors, frame->filter.words.ptr, frame->filter.words.len, &some, &run, + frame->error)) + return false; + if (!some) + { + *frame->error = txn_spill_internal("a carry's file wrote no key"); + return false; + } + file = (TxnSpillFile) + { + .xid = layered->xid, .kind = kind, .number = number, .len = run.len, .hash = run.hash, + .bits = txn_spill_Filter_bits(&frame->filter), .passed = passed + }; + if (!txn_spill_carried_push(frame, &frame->files, &file, sizeof(file), _Alignof(TxnSpillFile))) + return false; + if (frame->tuples.len > 0) + { + TxnSpillFile tupled; + StdVec tids; + + frame->tuples_live = false; + if (!txn_spill_tupled(&TXN_SPILL_STATE_HOME_OPS, doors, record, &frame->storage, &frame->dir, layered->xid, + number, &frame->tuples, &tupled, &tids, frame->error)) + return false; + std_vec_free_Checked(&tids, sizeof(uint64), _Alignof(uint64)); + tupled.passed = passed; + if (!txn_spill_carried_push(frame, &frame->files, &tupled, sizeof(tupled), _Alignof(TxnSpillFile))) + return false; + } + counted = (std_TupleU64U64_t) + { + layered->xid, count + }; + if (!txn_spill_carried_push(frame, &frame->counts, &counted, sizeof(counted), _Alignof(std_TupleU64U64_t))) + return false; + txn_spill_carried_file_drop(frame); + return true; } -/* - * txn_spill_files returns a storage's spill files of a kind's tree: those - * processes' slots name first, then the listed newest-listed first, each - * once, and, for a reader that looks for a key, only the listed whose - * filters it hits. - */ +/* txn_spill_carried_slots resolves each carried writer's status entry's slot before the carry's hold, as an entry with a count never moves or leaves. */ static bool -txn_spill_files(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, const HostStorage *storage, - TxnRunsKind kind, bool has_key, const uint8 *key, size_t key_len, StdVec *result, HostError *error) +txn_spill_carried_slots(txn_spill_carried_frame_0_t *frame) { - txn_spill_files_frame_0_t frame = {.home = home, .io = io, .state = state, .storage = storage, .kind = kind, - .has_key = has_key, .key = key, .key_len = key_len, .result = result, .error = error}; + std_vec_new(&frame->slots, sizeof(TxnSpillSlot), _Alignof(TxnSpillSlot)); + frame->slots_live = true; + if (!std_vec_try_reserve_exact(&frame->slots, frame->writers_len, sizeof(TxnSpillSlot), _Alignof(TxnSpillSlot))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + for (size_t k = 0; k < frame->writers_len; k++) + { + TxnMemtableVersionWriter writer = {.xid = frame->writers[k]}; + TxnSpillSlot slot = {.xid = frame->writers[k]}; - std_landing(&frame, txn_spill_files_body_0, txn_spill_files_pad_0); - return frame.ok; + if (!txn_status_kept_roomed(frame->io, &writer, &slot.has, &slot.roomed, frame->error)) + return false; + std_vec_push_Checked(&frame->slots, &slot, sizeof(slot), _Alignof(TxnSpillSlot)); + } + return true; } -/* txn_spill_looked_up_captures_t is a lookup's captures: the key and the records it answers. */ -typedef struct txn_spill_looked_up_captures_t +/* txn_spill_u64_cmp orders u64s ascending. */ +static int +txn_spill_u64_cmp(const void *a, const void *b) { - const uint8 *key; - size_t key_len; - StdVec *records; - bool *records_live; -} txn_spill_looked_up_captures_t; + uint64 x = *(const uint64 *) a; + uint64 y = *(const uint64 *) b; -/* txn_spill_looked_up_closure_0 is a lookup's read: the key's records in the file. */ + return x < y ? -1 : x > y; +} + +/* txn_spill_carried_ends gathers the ends of the carry's records from its chains' versions, ascending and each once. */ static bool -txn_spill_looked_up_closure_0(void *captures, HostIoIo *io, TxnSpillRead *read, HostError *error) +txn_spill_carried_ends(txn_spill_carried_frame_0_t *frame) { - txn_spill_looked_up_captures_t *c = captures; + size_t kept = 0; - if (!txn_spill_Read_looked_up(read, io, c->key, c->key_len, c->records, error)) - return false; - *c->records_live = true; + std_vec_new(&frame->ends, sizeof(uint64), _Alignof(uint64)); + frame->ends_live = true; + for (size_t k = 0; k < frame->chains.len; k++) + { + const TxnSpillChain *chain = &STD_VEC_AT(&frame->chains, TxnSpillChain, k); + + for (size_t v = 0; v < chain->versions.len; v++) + { + const txn_TupleU64VecU8_t *version = &STD_VEC_AT(&chain->versions, txn_TupleU64VecU8_t, v); + TxnMemtableVersionRecord stored = txn_memtable_version_Record_of(version->f1.ptr, version->f1.len); + uint64 lsn; + + if (!txn_memtable_version_Record_lsn(&stored, &lsn)) + continue; + if (!std_vec_try_reserve(&frame->ends, 1, sizeof(uint64), _Alignof(uint64))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(&frame->ends, &lsn, sizeof(uint64), _Alignof(uint64)); + } + } + if (frame->ends.len > 1) + qsort(frame->ends.ptr, frame->ends.len, sizeof(uint64), txn_spill_u64_cmp); + for (size_t k = 0; k < frame->ends.len; k++) + if (kept == 0 || STD_VEC_AT(&frame->ends, uint64, kept - 1) != STD_VEC_AT(&frame->ends, uint64, k)) + STD_VEC_AT(&frame->ends, uint64, kept++) = STD_VEC_AT(&frame->ends, uint64, k); + frame->ends.len = kept; return true; } -/* txn_spill_versions_frame_0_t is versions' state across its pad: the readings, the spills' directory, a file's path and records, and the versions found, while live. */ -typedef struct txn_spill_versions_frame_0_t -{ - const TxnSpillHomeOps *home; - HostIoIo *io; - HostSharedRecord state; - TxnRunsKind kind; - const uint8 *key; - size_t key_len; - StdVec *result; - HostError *error; - bool ok; - bool result_live; - HostStorage storage; - StdVec files; - bool files_live; - StdPathBuf dir; - bool dir_live; - StdString name; - bool name_live; - StdPathBuf path; - bool path_live; - /* records is a Vec>, of which moved are taken. */ - StdVec records; - bool records_live; - size_t moved; -} txn_spill_versions_frame_0_t; +/* txn_spill_unlist_pauses are the pauses an unlist's window runs, loaded before the carry's first count's add. */ +static const char *const txn_spill_unlist_pauses[] = {"doltgres-carry-unlisting", "doltgres-spill-manifest-opened", +"doltgres-spill-manifest-read"}; -/* txn_spill_versions_pad_0 is versions' pad: what it holds, while live. */ -static void -txn_spill_versions_pad_0(void *frame_) +/* + * txn_spill_carried_made keeps the files the carry listed and, for a logged + * storage, its unlist's record, both made before an unlist's window so the + * window allocates neither. + */ +static bool +txn_spill_carried_made(txn_spill_carried_frame_0_t *frame) { - txn_spill_versions_frame_0_t *frame = frame_; + size_t made = frame->files.len - frame->made_from; - if (frame->records_live) - { - frame->records_live = false; - for (size_t k = frame->moved; k < frame->records.len; k++) - std_vec_free_Checked(&STD_VEC_AT(&frame->records, StdVec, k), 1, 1); - std_vec_free_Checked(&frame->records, sizeof(StdVec), _Alignof(StdVec)); - } - if (frame->path_live) - { - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - } - if (frame->name_live) - { - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - } - if (frame->dir_live) + std_vec_new(&frame->made, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + frame->made_live = true; + if (!std_vec_try_reserve_exact(&frame->made, made, sizeof(TxnSpillFile), _Alignof(TxnSpillFile))) { - frame->dir_live = false; - std_vec_free_Checked(&frame->dir.bytes, 1, 1); - } - if (frame->files_live) - { - frame->files_live = false; - std_vec_free_Checked(&frame->files, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); + *frame->error = host_Error_out_of_memory(); + return false; } - if (frame->result_live) + std_vec_extend_from_slice_Checked(&frame->made, &STD_VEC_AT(&frame->files, TxnSpillFile, frame->made_from), made, + sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (frame->ends.len > 0) { - frame->result_live = false; - txn_memtable_version_versions_drop(frame->result, 0); + TxnRecordCarryUnlisted unlisted = {.storage = frame->storage, .ends = frame->ends}; + + txn_record_CarryUnlisted_encode(&unlisted, &frame->body); + frame->body_live = true; } + return true; } -/* txn_spill_versions_file reads one file's versions of the key into the frame's result, each by its handle. */ +/* + * txn_spill_carried_counted adds an amount to a listed carried file's + * writer's count through the slot resolved before the hold; a writer below + * the first normal xid, or with no entry, takes none. + */ static bool -txn_spill_versions_file(txn_spill_versions_frame_0_t *frame, const TxnSpillReading *reading) +txn_spill_carried_counted(txn_spill_carried_frame_0_t *frame, const std_TupleU64U64_t *counted, uint64 amount) { - TxnRunsNames names; - txn_spill_looked_up_captures_t captures = {frame->key, frame->key_len, &frame->records, &frame->records_live}; + const TxnSpillSlot *slot = NULL; - if (!txn_spill_names(frame->home, frame->state, &frame->storage, reading->kind, reading->xid, &names, - frame->error)) - return false; - txn_runs_spill_name(&names, reading->number, &frame->name); - frame->name_live = true; - std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, - &frame->path); - frame->path_live = true; - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - frame->moved = 0; - if (!txn_spill_reading(frame->io, &frame->path, &names, reading->has_len, reading->len, - txn_spill_looked_up_closure_0, &captures, frame->error)) - return false; - if (!std_vec_try_reserve(frame->result, frame->records.len, sizeof(TxnMemtableVersionVersion), - _Alignof(TxnMemtableVersionVersion))) - { - *frame->error = host_Error_out_of_memory(); - return false; - } - while (frame->moved < frame->records.len) + if (counted->f1 == 0 || counted->f0 < TXN_STATUS_FIRST_NORMAL) + return true; + for (size_t k = 0; k < frame->slots.len && slot == NULL; k++) + if (STD_VEC_AT(&frame->slots, TxnSpillSlot, k).xid == counted->f0) + slot = &STD_VEC_AT(&frame->slots, TxnSpillSlot, k); + + /* + * The rewrite's AccessExclusiveLock, taken before its caller read the + * writers, keeps the old storage from gaining a writer the carry spills + * under. + */ + Assert(slot != NULL); + if (slot == NULL || !slot->has) + return true; + return txn_status_counted(HOST_IO_IO_TAKES_SHORT(frame->io), &frame->hold, slot->roomed, amount, frame->error); +} + +/* txn_spill_carried_unlisted_closure_0 is the unlist's removal's test: one of the files the carry listed. */ +static bool +txn_spill_carried_unlisted_closure_0(void *captures, const TxnSpillFile *file) +{ + const StdVec *made = captures; + + for (size_t k = 0; k < made->len; k++) + if (txn_spill_File_PartialEq_eq(&STD_VEC_AT(made, TxnSpillFile, k), file)) + return true; + return false; +} + +/* + * txn_spill_carried_unlisted takes the carry's files back out of the + * manifest after the add of the count of file at failed did, in one + * checkpoint window, Postgres' critical section, so none of its steps can + * leave part of it done: it logs the unlist and flushes its record before + * the manifest loses a file, as RelationTruncate flushes before a truncate, + * takes the files out, and gives back the counts of the files before failed. + * A step's failure leaves the window open, which stops the server, as an + * error in a critical section does; its restart replays the carry's records + * and the unlist. + */ +static void +txn_spill_carried_unlisted(txn_spill_carried_frame_0_t *frame, size_t failed) +{ + HostSharedRecord record = host_shared_Held_record(frame->held); + StdVec gone; + uint64 lsn; + + if (!host_delay_checkpoint(HOST_IO_IO_IO_DOORS(frame->io), txn_spill_unlist_pauses, + lengthof(txn_spill_unlist_pauses), &frame->window, frame->error)) + return; + frame->window_live = true; + if (frame->body_live && + (!host_wal_insert(HOST_IO_IO_IO_DOORS(frame->io), HOST_WAL_KIND_CARRY_UNLISTED, frame->body.ptr, frame->body.len, + &lsn, frame->error) || + !host_wal_flush(HOST_IO_IO_IO_DOORS(frame->io), lsn, frame->error))) + return; + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-carry-unlisting", frame->error) || + !txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, frame->io, record, &frame->storage, + txn_spill_carried_unlisted_closure_0, &frame->made, &gone, frame->error)) + return; + std_vec_free_Checked(&gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + for (size_t k = 0; k < failed; k++) { - uint32 place = (uint32) frame->moved; - TxnMemtableVersionVersion version; + const std_TupleU64U64_t *counted = &STD_VEC_AT(&frame->counts, std_TupleU64U64_t, k); - version = txn_memtable_version_Version_spilled(txn_memtable_version_Handle_spilled(reading->xid, - reading->number, place), - &STD_VEC_AT(&frame->records, StdVec, frame->moved)); - frame->moved++; - std_vec_push_Checked(frame->result, &version, sizeof(version), _Alignof(TxnMemtableVersionVersion)); + if (!txn_spill_carried_counted(frame, counted, std_wrapping_sub_u64(0, counted->f1))) + return; } - frame->records_live = false; - std_vec_free_Checked(&frame->records, sizeof(StdVec), _Alignof(StdVec)); - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - return true; + frame->window_live = false; + host_DelayCheckpoint_finish(&frame->window); } -/* txn_spill_versions_body_0 is versions' body. */ +/* txn_spill_carried_body_0 is carried's body. */ static void -txn_spill_versions_body_0(void *frame_) +txn_spill_carried_body_0(void *frame_) { - txn_spill_versions_frame_0_t *frame = frame_; - HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + txn_spill_carried_frame_0_t *frame = frame_; + HostSharedRecord record = host_shared_Held_record(frame->held); + static const TxnWrittenOf ofs[3] = {TXN_WRITTEN_OF_ROWS, TXN_WRITTEN_OF_FREE, TXN_WRITTEN_OF_VALUES}; TxnStatePersistence persistence; + StdStr parent; + ChunksErrorError failed; - std_vec_new(frame->result, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); - if ((host_shared_Record_atomic(frame->state, frame->home->pended, load) == 0 && - host_shared_Record_atomic(frame->state, frame->home->listed, load) == 0) || - !frame->home->described(frame->state, &frame->storage, &persistence)) + if (!txn_state_described(record, &frame->storage, &persistence)) { frame->ok = true; return; } - frame->result_live = true; - txn_runs_spill_directory(frame->storage.database, frame->home->own(frame->state), &frame->dir); + for (size_t k = 0; k < lengthof(ofs); k++) + if (!txn_spill_State_Home_kind(frame->io, frame->held, ofs[k], &frame->kinds[k], &frame->tails[k], + frame->error)) + return; + txn_runs_spill_directory(frame->storage.database, txn_state_own(record), &frame->dir); frame->dir_live = true; - if (!txn_spill_files(frame->home, frame->io, frame->state, &frame->storage, frame->kind, true, frame->key, - frame->key_len, &frame->files, frame->error)) + if (!std_path_parent(frame->dir.bytes.ptr, frame->dir.bytes.len, &parent)) { - txn_spill_versions_pad_0(frame); + parent.ptr = frame->dir.bytes.ptr; + parent.len = frame->dir.bytes.len; + } + if (!chunks_file_create_dir(HOST_IO_IO_IO_DOORS(frame->io), parent.ptr, parent.len, &failed) || + !chunks_file_create_dir(HOST_IO_IO_IO_DOORS(frame->io), frame->dir.bytes.ptr, frame->dir.bytes.len, &failed)) + { + *frame->error = txn_base_storage(&failed); + txn_spill_carried_pad_0(frame); + return; + } + txn_spill_manifest(&TXN_SPILL_STATE_HOME_OPS, record, &frame->storage, &frame->path); + frame->path_live = true; + if (!txn_spill_pended(&TXN_SPILL_STATE_HOME_OPS, record, &frame->storage, &frame->slotted, frame->error)) + { + txn_spill_carried_pad_0(frame); + return; + } + frame->slotted_live = true; + if (!txn_spill_carried_slots(frame)) + { + txn_spill_carried_pad_0(frame); + return; + } + if (!host_shared_Record_lock_long(record, HOST_IO_IO_TAKES_SHORT(frame->io), TXN_STATE_SPILLS_WRITING_INDEX, + &frame->writing, frame->error)) + { + txn_spill_carried_pad_0(frame); + return; + } + frame->writing_live = true; + frame->long_io = host_shared_LongGuard_io(&frame->writing); + std_vec_new(&frame->chains, sizeof(TxnSpillChain), _Alignof(TxnSpillChain)); + if (!frame->logged(frame->captures, &frame->long_io, &frame->chains, frame->error)) + { + for (size_t k = 0; k < frame->chains.len; k++) + txn_spill_Chain_drop(&STD_VEC_AT(&frame->chains, TxnSpillChain, k)); + std_vec_free_Checked(&frame->chains, sizeof(TxnSpillChain), _Alignof(TxnSpillChain)); + txn_spill_carried_pad_0(frame); + return; + } + if (!txn_spill_carried_ends(frame) || !txn_spill_layered(&frame->chains, &frame->layers, frame->error)) + { + txn_spill_carried_pad_0(frame); + return; + } + frame->layers_live = true; + if (!host_pause(HOST_SHARED_LONG_IO_PAUSES(&frame->long_io), "doltgres-carry-logged", frame->error) || + !txn_spill_read(HOST_IO_IO_DOORS_READ_DOORS(HOST_SHARED_LONG_IO_IO_DOORS(&frame->long_io)), + HOST_SHARED_LONG_IO_PAUSES(&frame->long_io), &frame->path, &frame->files, frame->error)) + { + txn_spill_carried_pad_0(frame); return; } frame->files_live = true; - if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-spill-listed", frame->error)) + frame->made_from = frame->files.len; + std_vec_new(&frame->counts, sizeof(std_TupleU64U64_t), _Alignof(std_TupleU64U64_t)); + frame->counts_live = true; + for (size_t d = 0; d < frame->layers.len; d++) { - txn_spill_versions_pad_0(frame); + const StdVec *layer = &STD_VEC_AT(&frame->layers, StdVec, d); + + for (size_t f = 0; f < layer->len; f++) + if (!txn_spill_carried_file(frame, &STD_VEC_AT(layer, TxnSpillLayerFile, f))) + { + txn_spill_carried_pad_0(frame); + return; + } + } + if (!txn_spill_carried_made(frame) || + !host_load_pauses(HOST_IO_IO_IO_DOORS(frame->io), txn_spill_unlist_pauses, lengthof(txn_spill_unlist_pauses), + frame->error) || + !host_delta_hold(frame->io, &frame->hold, frame->error)) + { + txn_spill_carried_pad_0(frame); return; } - for (size_t k = 0; k < frame->files.len; k++) - if (!txn_spill_versions_file(frame, &STD_VEC_AT(&frame->files, TxnSpillReading, k))) + frame->hold_live = true; + if (!host_pause(HOST_SHARED_LONG_IO_PAUSES(&frame->long_io), "doltgres-carry-written", frame->error) || + !txn_spill_rewrote(&TXN_SPILL_STATE_HOME_OPS, &frame->long_io, record, &frame->path, &frame->files, + frame->error)) + { + txn_spill_carried_pad_0(frame); + return; + } + frame->writing_live = false; + host_shared_LongGuard_drop(&frame->writing); + for (size_t k = 0; k < frame->counts.len; k++) + { + const std_TupleU64U64_t *counted = &STD_VEC_AT(&frame->counts, std_TupleU64U64_t, k); + + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-carry-counting", frame->error) || + !txn_spill_carried_counted(frame, counted, counted->f1)) { - txn_spill_versions_pad_0(frame); + txn_spill_carried_unlisted(frame, k); + txn_spill_carried_pad_0(frame); return; } - frame->result_live = false; + } frame->ok = true; - txn_spill_versions_pad_0(frame); + txn_spill_carried_pad_0(frame); } /* - * txn_spill_versions returns a key's versions in a storage's spills of a - * kind, newest first: those of the file a process's slot names, then those - * of the listed files whose filters the key hits, newest-listed first, each - * file once, read under a hold. + * txn_spill_carried writes a carry's versions of other writers into a + * rewrite's or a move's new storage as spill files under each writer's + * top-level xid (P2), disk to disk, so the carry never waits at the delta's + * budget on versions it can't spill: under one hold of the storage's + * manifest's writers' lock, taken before any record, logged logs their + * records and answers each tree's chains with every version's record, its + * LSN the record's end; their layers' files (layered) are written and + * synced, numbered through the state's numbered as a residual's are, a row + * file with its file of spilled tuple ids, and listed oldest layer first, + * newest last; after the hold each file's versions are counted at its + * writer's entry as their pushes would be, a lock's left out. A checkpoint + * whose redo point falls after the records names the files, since its naming + * takes the same lock (named); one before them replays the records, which + * redo's holds check applies once. A failed write raises its error, leaving + * the files unlisted for the sweep. */ bool -txn_spill_versions(const TxnSpillHomeOps *home, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, - TxnRunsKind kind, const uint8 *key, size_t key_len, StdVec *result, HostError *error) +txn_spill_carried(HostIoIo *io, const HostSharedHeld *held, const uint64 *writers, size_t writers_len, + bool (*logged) (void *captures, HostSharedLongIo *io, StdVec *chains, HostError *error), + void *captures, HostError *error) { - txn_spill_versions_frame_0_t frame = {.home = home, .io = io, .state = state, .kind = kind, .key = key, - .key_len = key_len, .result = result, .error = error}; + txn_spill_carried_frame_0_t frame = {.io = io, .held = held, .logged = logged, .captures = captures, + .error = error, .writers = writers, .writers_len = writers_len}; - std_landing(&frame, txn_spill_versions_body_0, txn_spill_versions_pad_0); + std_landing(&frame, txn_spill_carried_body_0, txn_spill_carried_pad_0); return frame.ok; } -/* txn_spill_next_captures_t is next's read's captures: the place, the way, and the key it answers. */ -typedef struct txn_spill_next_captures_t -{ - bool has_after; - const uint8 *after; - size_t after_len; - TxnMemtableTableDirection way; - bool *some; - txn_TupleVecU8VecVecU8_t *result; - bool *result_live; -} txn_spill_next_captures_t; - -/* txn_spill_next_closure_0 is next's read: the file's first key past the place. */ -static bool -txn_spill_next_closure_0(void *captures, HostIoIo *io, TxnSpillRead *read, HostError *error) +/* + * txn_spill_unlisted_redo_captures_t is an unlist's redo's walk's captures: + * the state, the tree walked, the carry's ends, ascending, and for the rows + * the tuple ids of the rows taken out, or for the free space map those rows' + * tuple ids, ascending, whose takes go too. + */ +typedef struct txn_spill_unlisted_redo_captures_t { - txn_spill_next_captures_t *c = captures; - - if (!txn_spill_Read_next(read, io, c->has_after, c->after, c->after_len, c->way, c->some, c->result, error)) - return false; - *c->result_live = *c->some; - return true; -} + HostSharedRecord record; + const TxnMemtableTableTreeOps *tree; + const StdVec *ends; + StdVec *tids; + const StdVec *carried; +} txn_spill_unlisted_redo_captures_t; -/* txn_spill_next_frame_0_t is next's state across its pad: the readings, the spills' directory, a file's path and key, and the key found, while live. */ -typedef struct txn_spill_next_frame_0_t +/* txn_spill_unlisted_redo_frame_t is the walk's state at one key across its pad: the hold, the key's versions, and the handles taken out, each while live. */ +typedef struct txn_spill_unlisted_redo_frame_t { - const TxnSpillHomeOps *home; + txn_spill_unlisted_redo_captures_t *walk; HostIoIo *io; - HostSharedRecord state; - TxnRunsKind kind; - bool has_after; - const uint8 *after; - size_t after_len; - TxnMemtableTableDirection way; - bool *some; - txn_TupleVecU8VecVersion_t *result; + const uint8 *key; + size_t key_len; + HostDeltaHold hold; + bool hold_live; + StdVec versions; + bool versions_live; + StdVec handles; + bool handles_live; + StdVec taken; + bool taken_live; HostError *error; bool ok; - bool result_live; - HostStorage storage; - StdVec files; - bool files_live; - StdPathBuf dir; - bool dir_live; - StdString name; - bool name_live; - StdPathBuf path; - bool path_live; - txn_TupleVecU8VecVecU8_t spilled; - bool spilled_live; - size_t moved; -} txn_spill_next_frame_0_t; +} txn_spill_unlisted_redo_frame_t; -/* txn_spill_next_spilled_drop drops a file's key next read, its records not moved, while live. */ -static void -txn_spill_next_spilled_drop(txn_spill_next_frame_0_t *frame) +/* txn_spill_ends_hold reports whether a carry's ends, ascending, hold an LSN. */ +static bool +txn_spill_ends_hold(const StdVec *ends, uint64 lsn) { - if (frame->spilled_live) + size_t low = 0; + size_t high = ends->len; + + while (low < high) { - frame->spilled_live = false; - std_vec_free_Checked(&frame->spilled.f0, 1, 1); - for (size_t k = frame->moved; k < frame->spilled.f1.len; k++) - std_vec_free_Checked(&STD_VEC_AT(&frame->spilled.f1, StdVec, k), 1, 1); - std_vec_free_Checked(&frame->spilled.f1, sizeof(StdVec), _Alignof(StdVec)); + size_t mid = low + (high - low) / 2; + uint64 at = STD_VEC_AT(ends, uint64, mid); + + if (at == lsn) + return true; + if (at < lsn) + low = mid + 1; + else + high = mid; } + return false; } -/* txn_spill_next_pad_0 is next's pad: what it holds, while live. */ +/* + * txn_spill_unlisted_redo_body takes out the key's versions whose LSN is one + * of the carry's ends, and in the free space map the takes of the carried + * rows too, with their counts. + */ static void -txn_spill_next_pad_0(void *frame_) +txn_spill_unlisted_redo_body(void *frame_) { - txn_spill_next_frame_0_t *frame = frame_; + txn_spill_unlisted_redo_frame_t *frame = frame_; + txn_spill_unlisted_redo_captures_t *walk = frame->walk; - txn_spill_next_spilled_drop(frame); - if (frame->path_live) - { - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - } - if (frame->name_live) - { - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - } - if (frame->dir_live) - { - frame->dir_live = false; - std_vec_free_Checked(&frame->dir.bytes, 1, 1); - } - if (frame->files_live) - { - frame->files_live = false; - std_vec_free_Checked(&frame->files, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); - } - if (frame->result_live) + if (!host_delta_hold(frame->io, &frame->hold, frame->error)) + return; + frame->hold_live = true; + if (!txn_memtable_table_chain(walk->tree, frame->io, &frame->hold, walk->record, frame->key, frame->key_len, + &frame->versions, frame->error)) + return; + frame->versions_live = true; + std_vec_new(&frame->handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + frame->handles_live = true; + for (size_t i = 0; i < frame->versions.len; i++) { - frame->result_live = false; - std_vec_free_Checked(&frame->result->f0, 1, 1); - txn_memtable_version_versions_drop(&frame->result->f1, 0); - } -} - -/* txn_spill_next_file reads one file's first key past the place and keeps it when it is nearer than the one found, or the same key. */ -static bool -txn_spill_next_file(txn_spill_next_frame_0_t *frame, const TxnSpillReading *reading) -{ - TxnRunsNames names; - bool some; - txn_spill_next_captures_t captures = {frame->has_after, frame->after, frame->after_len, frame->way, &some, - &frame->spilled, &frame->spilled_live}; - bool nearer = !frame->result_live; - bool same = false; + TxnMemtableVersionVersion *version = &STD_VEC_AT(&frame->versions, TxnMemtableVersionVersion, i); + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); + uint64 lsn; - if (!txn_spill_names(frame->home, frame->state, &frame->storage, reading->kind, reading->xid, &names, - frame->error)) - return false; - txn_runs_spill_name(&names, reading->number, &frame->name); - frame->name_live = true; - std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, - &frame->path); - frame->path_live = true; - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - frame->moved = 0; - if (!txn_spill_reading(frame->io, &frame->path, &names, reading->has_len, reading->len, txn_spill_next_closure_0, - &captures, frame->error)) - return false; - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - if (!some) - return true; - if (frame->result_live) - { - StdOrdering order = std_bytes_Ord_cmp(frame->spilled.f0.ptr, frame->spilled.f0.len, frame->result->f0.ptr, - frame->result->f0.len); + uint64 tid = txn_memtable_map_taken(txn_memtable_version_Record_op(&record)).f0; - nearer = frame->way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD ? order == STD_ORDERING_LESS : - order == STD_ORDERING_GREATER; - same = order == STD_ORDERING_EQUAL; - } - if (nearer) - { - if (frame->result_live) + if (!(txn_memtable_version_Record_lsn(&record, &lsn) && txn_spill_ends_hold(walk->ends, lsn)) && + !(walk->carried != NULL && tid != 0 && txn_spill_ends_hold(walk->carried, tid))) + continue; + if (walk->tids != NULL && txn_spill_op_tid(txn_memtable_version_Record_op(&record), &tid)) { - frame->result_live = false; - std_vec_free_Checked(&frame->result->f0, 1, 1); - txn_memtable_version_versions_drop(&frame->result->f1, 0); + if (!std_vec_try_reserve(walk->tids, 1, sizeof(uint64), _Alignof(uint64))) + { + *frame->error = host_Error_out_of_memory(); + return; + } + std_vec_push_Checked(walk->tids, &tid, sizeof(uint64), _Alignof(uint64)); } - frame->result->f0 = frame->spilled.f0; - std_vec_new(&frame->spilled.f0, 1, 1); - std_vec_new(&frame->result->f1, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); - frame->result_live = true; - } - if (nearer || same) - { - if (!std_vec_try_reserve(&frame->result->f1, frame->spilled.f1.len, sizeof(TxnMemtableVersionVersion), - _Alignof(TxnMemtableVersionVersion))) + if (!std_vec_try_reserve(&frame->handles, 1, sizeof(TxnMemtableVersionHandle), + _Alignof(TxnMemtableVersionHandle))) { *frame->error = host_Error_out_of_memory(); - return false; - } - while (frame->moved < frame->spilled.f1.len) - { - uint32 place = (uint32) frame->moved; - TxnMemtableVersionVersion version; - - version = txn_memtable_version_Version_spilled(txn_memtable_version_Handle_spilled(reading->xid, - reading->number, - place), - &STD_VEC_AT(&frame->spilled.f1, StdVec, frame->moved)); - frame->moved++; - std_vec_push_Checked(&frame->result->f1, &version, sizeof(version), _Alignof(TxnMemtableVersionVersion)); + return; } + std_vec_push_Checked(&frame->handles, &version->handle, sizeof(TxnMemtableVersionHandle), + _Alignof(TxnMemtableVersionHandle)); } - txn_spill_next_spilled_drop(frame); - return true; + if (frame->handles.len > 0) + { + if (!txn_memtable_table_removed(walk->tree, frame->io, walk->record, frame->key, frame->key_len, + frame->handles.ptr, frame->handles.len, &frame->taken, frame->error)) + return; + frame->taken_live = true; + } + frame->ok = true; } -/* txn_spill_next_body_0 is next's body. */ +/* txn_spill_unlisted_redo_pad frees what the walk holds live at its key. */ static void -txn_spill_next_body_0(void *frame_) +txn_spill_unlisted_redo_pad(void *frame_) { - txn_spill_next_frame_0_t *frame = frame_; - HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; - TxnStatePersistence persistence; + txn_spill_unlisted_redo_frame_t *frame = frame_; - *frame->some = false; - if ((host_shared_Record_atomic(frame->state, frame->home->pended, load) == 0 && - host_shared_Record_atomic(frame->state, frame->home->listed, load) == 0) || - !frame->home->described(frame->state, &frame->storage, &persistence)) - { - frame->ok = true; - return; - } - txn_runs_spill_directory(frame->storage.database, frame->home->own(frame->state), &frame->dir); - frame->dir_live = true; - if (!txn_spill_files(frame->home, frame->io, frame->state, &frame->storage, frame->kind, false, NULL, 0, - &frame->files, frame->error)) - { - txn_spill_next_pad_0(frame); - return; - } - frame->files_live = true; - for (size_t k = 0; k < frame->files.len; k++) - if (!txn_spill_next_file(frame, &STD_VEC_AT(&frame->files, TxnSpillReading, k))) - { - txn_spill_next_pad_0(frame); - return; - } - *frame->some = frame->result_live; - frame->result_live = false; - frame->ok = true; - txn_spill_next_pad_0(frame); + if (frame->taken_live) + std_vec_free_Checked(&frame->taken, sizeof(bool), _Alignof(bool)); + if (frame->handles_live) + std_vec_free_Checked(&frame->handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + if (frame->versions_live) + txn_memtable_version_versions_drop(&frame->versions, 0); + if (frame->hold_live) + host_delta_Hold_drop(&frame->hold); + frame->taken_live = frame->handles_live = frame->versions_live = frame->hold_live = false; +} + +/* txn_spill_unlisted_redo_closure_0 is the unlist's redo's walk's closure, of one key of a tree. */ +static bool +txn_spill_unlisted_redo_closure_0(void *captures, HostIoIo *io, const uint8 *key, size_t key_len, + const StdVec *values, size_t values_len, HostError *error) +{ + txn_spill_unlisted_redo_frame_t frame = {.walk = captures, .io = io, .key = key, .key_len = key_len, + .error = error}; + + std_landing(&frame, txn_spill_unlisted_redo_body, txn_spill_unlisted_redo_pad); + txn_spill_unlisted_redo_pad(&frame); + return frame.ok; } /* - * txn_spill_next returns the first key of a storage's spills of a kind past - * a place, the way a scan goes, with its versions, newest first, the newer - * files' first, or nothing past the last, read under a hold; with no place - * it starts at the first key or the last. + * txn_spill_unlisted_redone takes out of one of a storage's memtables each + * version whose LSN is one of an unlisted carry's ends, adding the rows' + * tuple ids to tids when given, and with the carried rows' tuple ids given, + * the free space takes of those rows too, which carry no LSN, each with its + * count, as its primary took the carry's files back out (bugs row 291's + * review R2, G1, and G2). */ -bool -txn_spill_next(const TxnSpillHomeOps *home, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, - TxnRunsKind kind, bool has_after, const uint8 *after, size_t after_len, TxnMemtableTableDirection way, - bool *some, txn_TupleVecU8VecVersion_t *result, HostError *error) +static bool +txn_spill_unlisted_redone(HostIoIo *io, const HostSharedHeld *held, const TxnMemtableTableTreeOps *tree, + const TxnRecordCarryUnlisted *unlisted, StdVec *tids, const StdVec *carried, + HostError *error) { - txn_spill_next_frame_0_t frame = {.home = home, .io = io, .state = state, .kind = kind, .has_after = has_after, - .after = after, .after_len = after_len, .way = way, .some = some, .result = result, .error = error}; + txn_spill_unlisted_redo_captures_t walk = {host_shared_Held_record(held), tree, &unlisted->ends, tids, carried}; - std_landing(&frame, txn_spill_next_body_0, txn_spill_next_pad_0); - return frame.ok; + return txn_memtable_table_walked_killed(tree, io, walk.record, txn_spill_unlisted_redo_closure_0, &walk, NULL, + error); } -/* txn_spill_held_frame_0_t is the state of a read of a key's spilled versions under a hold across its pad: the hold and the versions, while live. */ -typedef struct txn_spill_held_frame_0_t +/* + * txn_spill_unlisted_run_frame_t is an unlist's redo's state at one writer's + * run of a kind across its pad: the files listed, the run's files and its + * rows' tuple-id files, and what the joins leave out, each while live. + */ +typedef struct txn_spill_unlisted_run_frame_t { - const TxnSpillHomeOps *home; HostIoIo *io; - HostSharedRecord state; + HostSharedRecord record; + const HostStorage *storage; + const StdPathBuf *dir; + const StdVec *ends; + const StdVec *carried; + uint64 xid; TxnRunsKind kind; - const uint8 *key; - size_t key_len; - bool (*with) (void *captures, const StdVec *versions, HostError *error); - void *captures; + TxnRunsTail tail; HostError *error; bool ok; - HostDeltaHold hold; - bool hold_live; - StdVec versions; - bool versions_live; -} txn_spill_held_frame_0_t; + txn_TupleVecFileU64_t listed; + bool listed_live; + StdVec own; + bool own_live; + StdVec named; + bool named_live; + TxnSpillLeft left; + bool left_live; + StdVec gone; + bool gone_live; +} txn_spill_unlisted_run_frame_t; -/* txn_spill_held_pad_0 is the pad of a read under a hold: the versions, then the hold, while live. */ +/* txn_spill_unlisted_run_pad frees what the run holds live. */ static void -txn_spill_held_pad_0(void *frame_) +txn_spill_unlisted_run_pad(void *frame_) { - txn_spill_held_frame_0_t *frame = frame_; + txn_spill_unlisted_run_frame_t *frame = frame_; - if (frame->versions_live) - { - frame->versions_live = false; - txn_memtable_version_versions_drop(&frame->versions, 0); - } - if (frame->hold_live) - { - frame->hold_live = false; - host_delta_Hold_drop(&frame->hold); - } + if (frame->gone_live) + std_vec_free_Checked(&frame->gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (frame->left_live) + std_vec_free_Checked(&frame->left.tids, sizeof(uint64), _Alignof(uint64)); + if (frame->named_live) + std_vec_free_Checked(&frame->named, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (frame->own_live) + std_vec_free_Checked(&frame->own, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (frame->listed_live) + std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + frame->gone_live = frame->left_live = frame->named_live = frame->own_live = frame->listed_live = false; } -/* txn_spill_held_body_0 reads the key's spilled versions under a hold and runs the step on them. */ -static void -txn_spill_held_body_0(void *frame_) +/* txn_spill_unlisted_run_closure_0 is the run's removal's test: one of the files the run's listing replaces. */ +static bool +txn_spill_unlisted_run_closure_0(void *captures, const TxnSpillFile *file) { - txn_spill_held_frame_0_t *frame = frame_; - - if (!host_delta_hold(frame->io, &frame->hold, frame->error)) - return; - frame->hold_live = true; - if (!txn_spill_versions(frame->home, frame->io, &frame->hold, frame->state, frame->kind, frame->key, - frame->key_len, &frame->versions, frame->error)) - { - txn_spill_held_pad_0(frame); - return; - } - frame->versions_live = true; - frame->ok = frame->with(frame->captures, &frame->versions, frame->error); - txn_spill_held_pad_0(frame); + return txn_spill_replaced_by(captures, 1, file); } -/* txn_spill_held runs a step on a key's versions in a storage's spills of a kind, read under a hold it takes. */ -static bool -txn_spill_held(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, TxnRunsKind kind, const uint8 *key, - size_t key_len, bool (*with) (void *captures, const StdVec *versions, HostError *error), - void *captures, HostError *error) +/* txn_spill_tid_cmp orders tuple ids ascending. */ +static int +txn_spill_tid_cmp(const void *a, const void *b) { - txn_spill_held_frame_0_t frame = {.home = home, .io = io, .state = state, .kind = kind, .key = key, - .key_len = key_len, .with = with, .captures = captures, .error = error}; + uint64 x = *(const uint64 *) a; + uint64 y = *(const uint64 *) b; - std_landing(&frame, txn_spill_held_body_0, txn_spill_held_pad_0); - return frame.ok; + return x < y ? -1 : x > y; } -/* txn_spill_twinned_captures_t is twinned's and holds' step's captures: the record's bytes or the LSN, and the answer. */ -typedef struct txn_spill_twinned_captures_t -{ - const uint8 *bytes; - size_t bytes_len; - uint64 lsn; - bool *result; -} txn_spill_twinned_captures_t; - -/* txn_spill_twinned_closure_0 is twinned's step: whether a version's record is the bytes. */ -static bool -txn_spill_twinned_closure_0(void *captures, const StdVec *versions, HostError *error) +/* + * txn_spill_unlisted_run_body joins the writer's whole run of the kind, and + * for rows its tuple-id files of the same numbers, without the carry's + * versions and their rows' tuple ids; when it left out any, it lists the + * join in place of the run, as a merge lists its file, or takes the run out + * whole when nothing is left, and gives the dropped versions' count back at + * the writer's entry. + */ +static void +txn_spill_unlisted_run_body(void *frame_) { - txn_spill_twinned_captures_t *c = captures; + txn_spill_unlisted_run_frame_t *frame = frame_; + TxnRunsNames names; + TxnRunsNames tuple_names; + TxnRunsTail none = {.tag = TXN_RUNS_TAIL_NONE}; + TxnSpillFile listing[2] = {0}; + size_t listing_len = 1; + uint32 number = 0; + uint32 lowest = PG_UINT32_MAX; + txn_TupleU64KindU32OptionU32_t replacing; + HostXid xid = {frame->xid}; + bool has; + uint64 added; - *c->result = false; - for (size_t k = 0; k < versions->len && !*c->result; k++) + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->record, frame->storage, &frame->listed, + frame->error)) + return; + frame->listed_live = true; + std_vec_new(&frame->own, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + frame->own_live = true; + std_vec_new(&frame->named, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + frame->named_live = true; + for (size_t k = 0; k < frame->listed.f0.len; k++) { - TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(versions, - TxnMemtableVersionVersion, - k)); - StdBytes bytes = txn_memtable_version_Record_bytes(&record); + const TxnSpillFile *file = &STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k); + StdVec *into = file->kind == frame->kind ? &frame->own : + file->kind == TXN_RUNS_KIND_SPILLED_TUPLES && frame->kind == TXN_RUNS_KIND_SPILLED_ROWS ? &frame->named : + NULL; - *c->result = bytes.len == c->bytes_len && memcmp(bytes.ptr, c->bytes, bytes.len) == 0; + if (file->xid != frame->xid || into == NULL) + continue; + if (!std_vec_try_reserve(into, 1, sizeof(TxnSpillFile), _Alignof(TxnSpillFile))) + { + *frame->error = host_Error_out_of_memory(); + return; + } + std_vec_push_Checked(into, file, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); } - return true; + if (frame->own.len == 0) + { + frame->ok = true; + return; + } + for (size_t k = 0; k < frame->own.len; k++) + { + number = Max(number, STD_ADD(u32, STD_VEC_AT(&frame->own, TxnSpillFile, k).number, 1, 0, 0)); + lowest = Min(lowest, STD_VEC_AT(&frame->own, TxnSpillFile, k).number); + } + frame->left = (TxnSpillLeft) + { + .ends = frame->ends, .carried = frame->carried + }; + std_vec_new(&frame->left.tids, sizeof(uint64), _Alignof(uint64)); + frame->left_live = true; + if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, frame->record, frame->storage, frame->kind, frame->xid, &names, + frame->error) || + !txn_spill_joined(frame->io, frame->dir, &names, frame->tail, frame->own.ptr, frame->own.len, number, + &frame->left, &listing[0], frame->error)) + return; + if (frame->left.dropped == 0) + { + frame->ok = true; + return; + } + if (frame->left.tids.len > 1) + qsort(frame->left.tids.ptr, frame->left.tids.len, sizeof(uint64), txn_spill_tid_cmp); + if (frame->named.len > 0) + { + if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, frame->record, frame->storage, TXN_RUNS_KIND_SPILLED_TUPLES, + frame->xid, &tuple_names, frame->error) || + !txn_spill_joined(frame->io, frame->dir, &tuple_names, none, frame->named.ptr, frame->named.len, number, + &frame->left, &listing[1], frame->error)) + return; + listing_len = listing[1].len > 0 ? 2 : 1; + } + replacing.f0 = frame->xid; + replacing.f1 = frame->kind; + replacing.f2 = number; + replacing.has_f3 = true; + replacing.f3 = lowest; + if (listing[0].len == 0) + { + if (!txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->record, frame->storage, + txn_spill_unlisted_run_closure_0, &replacing, &frame->gone, frame->error)) + return; + frame->gone_live = true; + } + else if (!txn_spill_pend(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->record, frame->storage, &listing[0], + listing_len > 1, true, lowest, frame->error) || + !txn_spill_linked(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->record, frame->storage, listing, + listing_len, true, replacing, frame->error) || + !txn_spill_settled(&TXN_SPILL_STATE_HOME_OPS, frame->io, &frame->record, frame->error)) + return; + if (frame->xid >= TXN_STATUS_FIRST_NORMAL && + !txn_status_add(frame->io, xid, std_wrapping_sub_u64(0, frame->left.dropped), &has, &added, frame->error)) + return; + frame->ok = true; } -/* txn_spill_twinned reports whether a key's spills of a kind hold a version of a record's bytes. */ -bool -txn_spill_twinned(HostIoIo *io, HostSharedRecord state, TxnRunsKind kind, const uint8 *key, size_t key_len, - const uint8 *bytes, size_t bytes_len, bool *result, HostError *error) +/* txn_spill_unlisted_run does one writer's run of a kind of an unlist's redo, as its body does. */ +static bool +txn_spill_unlisted_run(HostIoIo *io, HostSharedRecord record, const HostStorage *storage, const StdPathBuf *dir, + const StdVec *ends, const StdVec *carried, StdVec *tids, uint64 xid, TxnRunsKind kind, + TxnRunsTail tail, HostError *error) { - txn_spill_twinned_captures_t captures = {bytes, bytes_len, 0, result}; + txn_spill_unlisted_run_frame_t frame = {.io = io, .record = record, .storage = storage, .dir = dir, .ends = ends, + .carried = carried, .xid = xid, .kind = kind, .tail = tail, .error = error}; - return txn_spill_held(&TXN_SPILL_STATE_HOME_OPS, io, state, kind, key, key_len, txn_spill_twinned_closure_0, - &captures, error); + std_landing(&frame, txn_spill_unlisted_run_body, txn_spill_unlisted_run_pad); + if (frame.ok && frame.left_live && tids != NULL) + std_vec_extend_from_slice_Checked(tids, frame.left.tids.ptr, frame.left.tids.len, sizeof(uint64), + _Alignof(uint64)); + txn_spill_unlisted_run_pad(&frame); + return frame.ok; } -/* txn_spill_holds_closure_0 is holds' step: whether a version's record carries the LSN. */ +/* + * txn_spill_unlisted_of redoes an unlist's spill files of one tree: each + * writer's run of its kind joined without the carry's versions, or for the + * free space map without the carried rows' takes, adding a rows run's + * dropped tuple ids to tids when given. + */ static bool -txn_spill_holds_closure_0(void *captures, const StdVec *versions, HostError *error) +txn_spill_unlisted_of(HostIoIo *io, const HostSharedHeld *held, const HostStorage *storage, const StdPathBuf *dir, + const TxnRecordCarryUnlisted *unlisted, TxnWrittenOf of, const StdVec *carried, StdVec *tids, + HostError *error) { - txn_spill_twinned_captures_t *c = captures; + HostSharedRecord record = host_shared_Held_record(held); + TxnRunsKind kind; + TxnRunsTail tail; + txn_TupleVecFileU64_t listed; + StdVec xids; + bool ok; - *c->result = false; - for (size_t k = 0; k < versions->len && !*c->result; k++) + if (!txn_spill_State_Home_kind(io, held, of, &kind, &tail, error) || + !txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, record, storage, &listed, error)) + return false; + std_vec_new(&xids, sizeof(uint64), _Alignof(uint64)); + for (size_t k = 0; k < listed.f0.len; k++) { - TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(versions, - TxnMemtableVersionVersion, - k)); - uint64 lsn; + const TxnSpillFile *file = &STD_VEC_AT(&listed.f0, TxnSpillFile, k); - *c->result = txn_memtable_version_Record_lsn(&record, &lsn) && lsn == c->lsn; + if (file->kind == kind && !txn_spill_has_xid(&xids, file->xid)) + std_vec_push_Checked(&xids, &file->xid, sizeof(uint64), _Alignof(uint64)); } - return true; + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + ok = true; + for (size_t k = 0; k < xids.len && ok; k++) + ok = txn_spill_unlisted_run(io, record, storage, dir, &unlisted->ends, carried, tids, + STD_VEC_AT(&xids, uint64, k), kind, tail, error); + std_vec_free_Checked(&xids, sizeof(uint64), _Alignof(uint64)); + return ok; } -/* txn_spill_holds reports whether a key's spills of a kind hold a version carrying the end of a WAL record. */ +/* + * txn_spill_redo_unlisted replays a carry's unlist (bugs row 291's review + * R2): it takes out of the storage's delta each version whose LSN is one of + * the record's ends, with its count, from the memtables and, for each + * writer's run of spill files of the rows, the free space map, or the value + * map holding one, by a join of the run without them listed in its place, + * and the free space takes of the rows it takes out, which carry no LSN, so + * a standby or a restart keeps none of a carry its primary took back. + */ bool -txn_spill_holds(HostIoIo *io, HostSharedRecord state, TxnRunsKind kind, const uint8 *key, size_t key_len, uint64 lsn, - bool *result, HostError *error) +txn_spill_redo_unlisted(HostIoIo *io, const HostSharedHeld *held, const TxnRecordCarryUnlisted *unlisted, + HostError *error) { - txn_spill_twinned_captures_t captures = {NULL, 0, lsn, result}; + HostSharedRecord record = host_shared_Held_record(held); + HostStorage storage; + TxnStatePersistence persistence; + StdPathBuf dir; + bool described = txn_state_described(record, &storage, &persistence); + bool ok; + StdVec tids; + size_t kept = 0; - return txn_spill_held(&TXN_SPILL_STATE_HOME_OPS, io, state, kind, key, key_len, txn_spill_holds_closure_0, - &captures, error); + std_vec_new(&tids, sizeof(uint64), _Alignof(uint64)); + if (described) + txn_runs_spill_directory(storage.database, txn_state_own(record), &dir); + ok = txn_spill_unlisted_redone(io, held, &TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, unlisted, &tids, NULL, error) && + (!described || txn_spill_unlisted_of(io, held, &storage, &dir, unlisted, TXN_WRITTEN_OF_ROWS, NULL, &tids, + error)); + if (ok && tids.len > 1) + qsort(tids.ptr, tids.len, sizeof(uint64), txn_spill_tid_cmp); + for (size_t k = 0; ok && k < tids.len; k++) + if (kept == 0 || STD_VEC_AT(&tids, uint64, kept - 1) != STD_VEC_AT(&tids, uint64, k)) + STD_VEC_AT(&tids, uint64, kept++) = STD_VEC_AT(&tids, uint64, k); + tids.len = kept; + ok = ok && txn_spill_unlisted_redone(io, held, &TXN_MEMTABLE_MAP_FREE_TREE_OPS, unlisted, NULL, &tids, error) && + txn_spill_unlisted_redone(io, held, &TXN_MEMTABLE_MAP_VALUES_TREE_OPS, unlisted, NULL, NULL, error) && + (!described || + (txn_spill_unlisted_of(io, held, &storage, &dir, unlisted, TXN_WRITTEN_OF_FREE, &tids, NULL, error) && + txn_spill_unlisted_of(io, held, &storage, &dir, unlisted, TXN_WRITTEN_OF_VALUES, NULL, NULL, error))); + std_vec_free_Checked(&tids, sizeof(uint64), _Alignof(uint64)); + if (described) + std_vec_free_Checked(&dir.bytes, 1, 1); + return ok; } -/* txn_spill_newest_value_captures_t is newest_value's step's captures: the entry it answers. */ -typedef struct txn_spill_newest_value_captures_t +/* txn_spill_unfolded_frame_0_t is unfolded_since's state across its pad: the newest base, the hold, and the next key found, while live. */ +typedef struct txn_spill_unfolded_frame_0_t { - bool *some; - TxnMemtableMapEntry *result; -} txn_spill_newest_value_captures_t; + HostIoIo *io; + const HostSharedHeld *held; + HostLsn end; + bool *result; + HostError *error; + bool ok; + TxnBaseBase base; + bool base_live; + HostDeltaHold hold; + bool hold_live; + txn_TupleVecU8VecVersion_t next; + bool next_live; + txn_TupleVecFileU64_t listed; + bool listed_live; +} txn_spill_unfolded_frame_0_t; -/* txn_spill_newest_value_closure_0 is newest_value's step: the newest version's put or removal. */ -static bool -txn_spill_newest_value_closure_0(void *captures, const StdVec *versions, HostError *error) +/* txn_spill_unfolded_pad_0 is unfolded_since's pad: what it holds, while live. */ +static void +txn_spill_unfolded_pad_0(void *frame_) { - txn_spill_newest_value_captures_t *c = captures; - TxnMemtableVersionRecord record; - TxnMemtableVersionOpRef op; + txn_spill_unfolded_frame_0_t *frame = frame_; - *c->some = versions->len > 0; - if (!*c->some) - return true; - record = txn_memtable_version_Version_record(&STD_VEC_AT(versions, TxnMemtableVersionVersion, 0)); - op = txn_memtable_version_Record_op(&record); - if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_PUT) + if (frame->listed_live) { - c->result->has_value = true; - std_slice_to_vec_Checked(&c->result->value, op.u.put.value.ptr, op.u.put.value.len, 1, 1); - return true; + frame->listed_live = false; + std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); } - if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_DELETE) + if (frame->next_live) { - c->result->has_value = false; - return true; + frame->next_live = false; + std_vec_free_Checked(&frame->next.f0, 1, 1); + txn_memtable_version_versions_drop(&frame->next.f1, 0); + } + if (frame->hold_live) + { + frame->hold_live = false; + host_delta_Hold_drop(&frame->hold); + } + if (frame->base_live) + { + frame->base_live = false; + txn_base_Base_drop(&frame->base); } - *c->some = false; - *error = txn_spill_internal("a spilled map's version neither puts nor deletes"); - return false; -} - -/* txn_spill_newest_value returns a value id's newest entry in a storage's spilled value maps: nothing for an id they hold no version of, and none for a removal. */ -bool -txn_spill_newest_value(HostIoIo *io, HostSharedRecord state, uint32 id, bool *some, TxnMemtableMapEntry *result, - HostError *error) -{ - uint8 key[4]; - txn_spill_newest_value_captures_t captures = {some, result}; - - std_u32_to_be_bytes(id, key); - return txn_spill_held(&TXN_SPILL_STATE_HOME_OPS, io, state, TXN_RUNS_KIND_SPILLED_VALUES, key, sizeof(key), - txn_spill_newest_value_closure_0, &captures, error); } -/* txn_spill_generation returns the generation of a storage's spills' manifest, which every change of it moves. */ -uint64 -txn_spill_generation(const TxnSpillHomeOps *home, HostSharedRecord state) +/* txn_spill_unfolded_body_0 is unfolded_since's body. */ +static void +txn_spill_unfolded_body_0(void *frame_) { + txn_spill_unfolded_frame_0_t *frame = frame_; + HostSharedRecord record = host_shared_Held_record(frame->held); HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + bool some; + uint64 word; - return host_shared_Record_atomic(state, home->changed, load); -} - -/* txn_spill_chunk_versions returns a key's versions in a working state's chunk spills, newest first, as versions does. */ -bool -txn_spill_chunk_versions(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord deltas, const uint8 *key, - size_t key_len, StdVec *result, HostError *error) -{ - return txn_spill_versions(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, io, hold, deltas, TXN_RUNS_KIND_SPILLED_CHUNKS, key, - key_len, result, error); -} - -/* txn_spill_chunk_closure_0 is chunk's step: the newest spilled version's chunk. */ -static bool -txn_spill_chunk_closure_0(void *captures, const StdVec *versions, HostError *error) -{ - txn_spill_newest_value_captures_t *c = captures; - TxnMemtableVersionRecord record; - TxnMemtableVersionOpRef op; + *frame->result = false; + if (!txn_state_newest(frame->io, frame->held, &some, &frame->base, frame->error)) + return; + frame->base_live = some; + if (some && frame->base.read >= frame->end) + { + frame->ok = true; + txn_spill_unfolded_pad_0(frame); + return; + } + if (host_shared_Record_atomic(record, TXN_STATE_SPILLS_PENDING_INDEX, load) != 0 || + txn_memtable_table_memtable(&TXN_MEMTABLE_MAP_VALUES_TREE_OPS, record, &word)) + { + *frame->result = true; + frame->ok = true; + txn_spill_unfolded_pad_0(frame); + return; + } + if (host_shared_Record_atomic(record, TXN_STATE_SPILLS_INDEX, load) != 0) + { + HostStorage storage; + TxnStatePersistence persistence; - *c->some = versions->len > 0; - if (!*c->some) - return true; - record = txn_memtable_version_Version_record(&STD_VEC_AT(versions, TxnMemtableVersionVersion, 0)); - op = txn_memtable_version_Record_op(&record); - if (op.tag != TXN_MEMTABLE_VERSION_OP_REF_PUT) + if (txn_state_described(record, &storage, &persistence)) + { + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, frame->io, record, &storage, &frame->listed, + frame->error)) + { + txn_spill_unfolded_pad_0(frame); + return; + } + frame->listed_live = true; + for (size_t k = 0; k < frame->listed.f0.len && !*frame->result; k++) + *frame->result = STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k).kind != TXN_RUNS_KIND_SPILLED_FREE; + } + if (*frame->result) + { + frame->ok = true; + txn_spill_unfolded_pad_0(frame); + return; + } + } + if (!host_delta_hold(frame->io, &frame->hold, frame->error)) { - *c->some = false; - *error = txn_spill_internal("a spilled chunk's version puts no chunk"); - return false; + txn_spill_unfolded_pad_0(frame); + return; } - std_slice_to_vec_Checked(&c->result->value, op.u.put.value.ptr, op.u.put.value.len, 1, 1); - return true; + frame->hold_live = true; + if (!txn_memtable_table_next(frame->io, &frame->hold, record, false, NULL, 0, TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, + &some, &frame->next, frame->error)) + { + txn_spill_unfolded_pad_0(frame); + return; + } + frame->next_live = some; + *frame->result = some; + frame->ok = true; + txn_spill_unfolded_pad_0(frame); } /* - * txn_spill_chunk returns the bytes of the chunk under a key in a working - * state's delta, else in its chunk spills, the newest version's, or nothing - * for a key neither holds. + * txn_spill_unfolded_since reports whether a storage state's newest base is + * read below a commit's end while the state holds a row, entry, or value map + * version that no fold applied, in its memtables or its spills, which may + * name a chunk of that commit's transaction; its free space map's versions + * name none. */ -bool -txn_spill_chunk(HostIoIo *io, HostSharedRecord deltas, const uint8 *key, size_t key_len, bool *some, StdVec *result, - HostError *error) +static bool +txn_spill_unfolded_since(HostIoIo *io, const HostSharedHeld *held, HostLsn end, bool *result, HostError *error) { - TxnMemtableMapEntry entry; - txn_spill_newest_value_captures_t captures = {some, &entry}; + txn_spill_unfolded_frame_0_t frame = {.io = io, .held = held, .end = end, .result = result, .error = error}; - if (!txn_memtable_branch_get(io, deltas, key, key_len, some, result, error)) - return false; - if (*some) - return true; - if (!txn_spill_held(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, io, deltas, TXN_RUNS_KIND_SPILLED_CHUNKS, key, key_len, - txn_spill_chunk_closure_0, &captures, error)) - return false; - if (*some) - *result = entry.value; - return true; + std_landing(&frame, txn_spill_unfolded_body_0, txn_spill_unfolded_pad_0); + return frame.ok; } -/* txn_spill_tuple_key_frame_0_t is tuple_key's state across its pad: the readings, the spills' directory, a file's path, and the values read, while live. */ -typedef struct txn_spill_tuple_key_frame_0_t +/* txn_spill_chunks_ended_frame_0_t is chunks_ended's state across its pad: the files listed, the xids seen and ended, the database's states, and the files removed, while live. */ +typedef struct txn_spill_chunks_ended_frame_0_t { HostIoIo *io; - HostSharedRecord state; - HostRow tid; - bool *some; - StdVec *result; + HostSharedRecord deltas; HostError *error; bool ok; HostStorage storage; - StdVec files; - bool files_live; - StdPathBuf dir; - bool dir_live; - StdString name; - bool name_live; - StdPathBuf path; - bool path_live; - /* values is a Vec>. */ - StdVec values; - bool values_live; -} txn_spill_tuple_key_frame_0_t; + txn_TupleVecFileU64_t listed; + bool listed_live; + /* seen and ended are Vecs of xids. */ + StdVec seen; + bool seen_live; + StdVec ended; + bool ended_live; + /* states is a Vec<(u64, Held)>. */ + StdVec states; + bool states_live; + StdVec gone; + bool gone_live; +} txn_spill_chunks_ended_frame_0_t; -/* txn_spill_tuple_key_pad_0 is tuple_key's pad: what it holds, while live. */ +/* txn_spill_states_drop drops a Vec<(u64, Held)>. */ static void -txn_spill_tuple_key_pad_0(void *frame_) +txn_spill_states_drop(StdVec *states) { - txn_spill_tuple_key_frame_0_t *frame = frame_; + for (size_t k = 0; k < states->len; k++) + host_shared_Held_drop(&STD_VEC_AT(states, txn_TupleU64HeldState_t, k).f1); + std_vec_free_Checked(states, sizeof(txn_TupleU64HeldState_t), _Alignof(txn_TupleU64HeldState_t)); +} - if (frame->values_live) +/* txn_spill_chunks_ended_pad_0 is chunks_ended's pad: what it holds, while live. */ +static void +txn_spill_chunks_ended_pad_0(void *frame_) +{ + txn_spill_chunks_ended_frame_0_t *frame = frame_; + + if (frame->gone_live) { - frame->values_live = false; - txn_spill_versions_drop(&frame->values); + frame->gone_live = false; + std_vec_free_Checked(&frame->gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); } - if (frame->path_live) + if (frame->states_live) { - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); + frame->states_live = false; + txn_spill_states_drop(&frame->states); } - if (frame->name_live) + if (frame->ended_live) { - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); + frame->ended_live = false; + std_vec_free_Checked(&frame->ended, sizeof(uint64), _Alignof(uint64)); } - if (frame->dir_live) + if (frame->seen_live) { - frame->dir_live = false; - std_vec_free_Checked(&frame->dir.bytes, 1, 1); + frame->seen_live = false; + std_vec_free_Checked(&frame->seen, sizeof(uint64), _Alignof(uint64)); } - if (frame->files_live) + if (frame->listed_live) { - frame->files_live = false; - std_vec_free_Checked(&frame->files, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); + frame->listed_live = false; + std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); } } -/* txn_spill_tuple_key_body_0 is tuple_key's body. */ +/* txn_spill_chunks_ended_gone reports whether a committed transaction's chunk spills may go: its status entry gone, or no state of the repository unfolded past its commit. */ +static bool +txn_spill_chunks_ended_gone(txn_spill_chunks_ended_frame_0_t *frame, HostXid xid, bool *gone) +{ + TxnStatusCommit commit; + + if (!txn_status_committed(frame->io, xid, &commit, frame->error)) + return false; + *gone = commit.tag == TXN_STATUS_COMMIT_MISSING; + if (commit.tag != TXN_STATUS_COMMIT_AT) + return true; + if (!txn_state_states(frame->io, frame->storage.database, &frame->states, frame->error)) + return false; + frame->states_live = true; + *gone = true; + for (size_t k = 0; k < frame->states.len && *gone; k++) + { + bool unfolded; + + if (!txn_spill_unfolded_since(frame->io, &STD_VEC_AT(&frame->states, txn_TupleU64HeldState_t, k).f1, + commit.u.at, &unfolded, frame->error)) + return false; + *gone = !unfolded; + } + frame->states_live = false; + txn_spill_states_drop(&frame->states); + return true; +} + +/* txn_spill_chunks_ended_body_0 is chunks_ended's body. */ static void -txn_spill_tuple_key_body_0(void *frame_) +txn_spill_chunks_ended_body_0(void *frame_) { - txn_spill_tuple_key_frame_0_t *frame = frame_; - HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + txn_spill_chunks_ended_frame_0_t *frame = frame_; TxnStatePersistence persistence; - uint8 key[8]; - *frame->some = false; - if ((host_shared_Record_atomic(frame->state, TXN_STATE_SPILLS_PENDING_INDEX, load) == 0 && - host_shared_Record_atomic(frame->state, TXN_STATE_SPILLS_INDEX, load) == 0) || - !txn_state_described(frame->state, &frame->storage, &persistence)) + if (!txn_spill_BranchDeltas_Home_described(frame->deltas, &frame->storage, &persistence)) { frame->ok = true; return; } - txn_runs_spill_directory(frame->storage.database, txn_state_own(frame->state), &frame->dir); - frame->dir_live = true; - std_u64_to_be_bytes(frame->tid.f0, key); - if (!txn_spill_files(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->state, &frame->storage, - TXN_RUNS_KIND_SPILLED_TUPLES, true, key, sizeof(key), &frame->files, frame->error)) - { - txn_spill_tuple_key_pad_0(frame); + if (!txn_spill_listed(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, frame->io, frame->deltas, &frame->storage, + &frame->listed, frame->error)) return; - } - frame->files_live = true; - for (size_t k = 0; k < frame->files.len; k++) + frame->listed_live = true; + std_vec_new(&frame->seen, sizeof(uint64), _Alignof(uint64)); + frame->seen_live = true; + std_vec_new(&frame->ended, sizeof(uint64), _Alignof(uint64)); + frame->ended_live = true; + for (size_t k = 0; k < frame->listed.f0.len; k++) { - const TxnSpillReading *reading = &STD_VEC_AT(&frame->files, TxnSpillReading, k); - TxnRunsNames names; - txn_spill_looked_up_captures_t captures = {key, sizeof(key), &frame->values, &frame->values_live}; + HostXid xid = {STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k).xid}; + HostStatus status; + bool gone = false; - if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, frame->state, &frame->storage, reading->kind, reading->xid, - &names, frame->error)) + if (txn_spill_has_xid(&frame->seen, xid.f0)) + continue; + if (!std_vec_try_reserve(&frame->seen, 1, sizeof(uint64), _Alignof(uint64))) { - txn_spill_tuple_key_pad_0(frame); + *frame->error = host_Error_out_of_memory(); + txn_spill_chunks_ended_pad_0(frame); return; } - txn_runs_spill_name(&names, reading->number, &frame->name); - frame->name_live = true; - std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, - &frame->path); - frame->path_live = true; - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - if (!txn_spill_reading(frame->io, &frame->path, &names, reading->has_len, reading->len, - txn_spill_looked_up_closure_0, &captures, frame->error)) + std_vec_push_Checked(&frame->seen, &xid.f0, sizeof(uint64), _Alignof(uint64)); + if (!host_status(HOST_IO_IO_IO_DOORS(frame->io), xid, &status, frame->error) || + (status == HOST_STATUS_COMMITTED && !txn_spill_chunks_ended_gone(frame, xid, &gone))) { - txn_spill_tuple_key_pad_0(frame); + txn_spill_chunks_ended_pad_0(frame); return; } - frame->values_live = true; - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - if (frame->values.len > 0) + if (status != HOST_STATUS_COMMITTED || !gone) + continue; + if (!std_vec_try_reserve(&frame->ended, 1, sizeof(uint64), _Alignof(uint64))) { - *frame->result = STD_VEC_AT(&frame->values, StdVec, 0); - for (size_t v = 1; v < frame->values.len; v++) - std_vec_free_Checked(&STD_VEC_AT(&frame->values, StdVec, v), 1, 1); - frame->values_live = false; - std_vec_free_Checked(&frame->values, sizeof(StdVec), _Alignof(StdVec)); - *frame->some = true; - break; + *frame->error = host_Error_out_of_memory(); + txn_spill_chunks_ended_pad_0(frame); + return; } - frame->values_live = false; - txn_spill_versions_drop(&frame->values); + std_vec_push_Checked(&frame->ended, &xid.f0, sizeof(uint64), _Alignof(uint64)); + } + if (frame->ended.len > 0) + { + if (!txn_spill_removed(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, frame->io, frame->deltas, &frame->storage, + txn_spill_xids_closure_0, &frame->ended, &frame->gone, frame->error)) + { + txn_spill_chunks_ended_pad_0(frame); + return; + } + frame->gone_live = true; } frame->ok = true; - txn_spill_tuple_key_pad_0(frame); + txn_spill_chunks_ended_pad_0(frame); } /* - * txn_spill_tuple_key returns the key of a tid's row in a storage's spilled - * tuple ids, the pending file's first, or nothing for a tid they don't hold, - * read under a hold. + * txn_spill_chunks_ended takes out of a working state's chunk spills' + * manifest the files of every committed transaction whose status entry is + * gone, or past whose commit no storage state of the repository holds an + * unfolded version that may name one of its chunks; a later checkpoint + * unlinks the files it took out. */ bool -txn_spill_tuple_key(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, HostRow tid, bool *some, - StdVec *result, HostError *error) +txn_spill_chunks_ended(HostIoIo *io, HostSharedRecord deltas, HostError *error) { - txn_spill_tuple_key_frame_0_t frame = {.io = io, .state = state, .tid = tid, .some = some, .result = result, - .error = error}; + txn_spill_chunks_ended_frame_0_t frame = {.io = io, .deltas = deltas, .error = error}; - std_landing(&frame, txn_spill_tuple_key_body_0, txn_spill_tuple_key_pad_0); + std_landing(&frame, txn_spill_chunks_ended_body_0, txn_spill_chunks_ended_pad_0); return frame.ok; } -/* TxnSpillNaming is Naming: a spill file a checkpoint names, its xid, kind, and number, and its length and xxh3 when its manifest lists them. */ -typedef struct TxnSpillNaming +/* txn_spill_identity_applied reports whether a save's fold of the identity tree at a point applies a transaction's identity spills: its commit record ends at or below the point. */ +bool +txn_spill_identity_applied(HostIoIo *io, uint64 xid, HostLsn point, bool *result, HostError *error) { - uint64 xid; - TxnRunsKind kind; - uint32 number; - bool has_read; - uint64 len; - uint64 hash; -} TxnSpillNaming; + HostXid of = {xid}; + HostStatus status; + TxnStatusCommit commit; -/* txn_spill_named_frame_0_t is named's state across its pad: the slots, the listed files, the namings, the aborted xids, the spills' directory, a file's path and file, and the named run files, while live. */ -typedef struct txn_spill_named_frame_0_t + *result = false; + if (!host_status(HOST_IO_IO_IO_DOORS(io), of, &status, error)) + return false; + if (status != HOST_STATUS_COMMITTED) + return true; + if (!txn_status_committed(io, of, &commit, error)) + return false; + *result = commit.tag == TXN_STATUS_COMMIT_AT && commit.u.at <= point; + return true; +} + +/* txn_spill_identity_applies_frame_0_t is identity_applies's state across its pad: the files listed, while live. */ +typedef struct txn_spill_identity_applies_frame_0_t { - const TxnSpillHomeOps *home; HostIoIo *io; - HostSharedRecord state; - const HostStorage *storage; - StdVec *result; + HostSharedRecord deltas; + HostLsn point; + bool *result; HostError *error; bool ok; - bool result_live; - StdVec slotted; - bool slotted_live; + HostStorage storage; txn_TupleVecFileU64_t listed; bool listed_live; - /* files is a Vec. */ - StdVec files; - bool files_live; - /* aborted is a Vec of xids. */ - StdVec aborted; - bool aborted_live; - StdVec gone; - bool gone_live; - StdPathBuf dir; - bool dir_live; - StdString name; - bool name_live; - StdPathBuf path; - bool path_live; - HostFile file; - bool file_live; -} txn_spill_named_frame_0_t; +} txn_spill_identity_applies_frame_0_t; -/* txn_spill_named_pad_0 is named's pad: what it holds, while live. */ +/* txn_spill_identity_applies_pad_0 is identity_applies's pad. */ static void -txn_spill_named_pad_0(void *frame_) +txn_spill_identity_applies_pad_0(void *frame_) { - txn_spill_named_frame_0_t *frame = frame_; + txn_spill_identity_applies_frame_0_t *frame = frame_; - if (frame->file_live) - { - frame->file_live = false; - host_File_drop(&frame->file); - } - if (frame->path_live) - { - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - } - if (frame->name_live) - { - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - } - if (frame->dir_live) - { - frame->dir_live = false; - std_vec_free_Checked(&frame->dir.bytes, 1, 1); - } - if (frame->gone_live) - { - frame->gone_live = false; - std_vec_free_Checked(&frame->gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - } - if (frame->aborted_live) - { - frame->aborted_live = false; - std_vec_free_Checked(&frame->aborted, sizeof(uint64), _Alignof(uint64)); - } - if (frame->files_live) - { - frame->files_live = false; - std_vec_free_Checked(&frame->files, sizeof(TxnSpillNaming), _Alignof(TxnSpillNaming)); - } if (frame->listed_live) - { - frame->listed_live = false; std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - } - if (frame->slotted_live) - { - frame->slotted_live = false; - std_vec_free_Checked(&frame->slotted, sizeof(txn_TupleU64KindU32OptionU32_t), - _Alignof(txn_TupleU64KindU32OptionU32_t)); - } - if (frame->result_live) - { - frame->result_live = false; - std_vec_free_Checked(frame->result, sizeof(RepoRunFile), _Alignof(RepoRunFile)); - } + frame->listed_live = false; } -/* txn_spill_named_push adds an element to a vector, refusing it when the vector can't grow. */ -static bool -txn_spill_named_push(txn_spill_named_frame_0_t *frame, StdVec *vector, const void *elem, size_t size, size_t align) +/* txn_spill_identity_applies_body_0 is identity_applies's body. */ +static void +txn_spill_identity_applies_body_0(void *frame_) { - if (!std_vec_try_reserve(vector, 1, size, align)) + txn_spill_identity_applies_frame_0_t *frame = frame_; + TxnStatePersistence persistence; + + *frame->result = false; + if (!txn_spill_Identity_Home_described(frame->deltas, &frame->storage, &persistence)) { - *frame->error = host_Error_out_of_memory(); - return false; + frame->ok = true; + return; } - std_vec_push_Checked(vector, elem, size, align); - return true; + if (!txn_spill_listed(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, frame->deltas, &frame->storage, &frame->listed, + frame->error)) + return; + frame->listed_live = true; + for (size_t k = 0; k < frame->listed.f0.len && !*frame->result; k++) + if (!txn_spill_identity_applied(frame->io, STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k).xid, frame->point, + frame->result, frame->error)) + return; + frame->ok = true; } -/* txn_spill_named_listed gathers the namings of the listed files, oldest-linked first, passing over those a pending merge replaces and gathering the aborted. */ -static bool -txn_spill_named_listed(txn_spill_named_frame_0_t *frame) +/* + * txn_spill_identity_applies reports whether a working state's identity + * spills list a file whose transaction's commit a save's fold at a point + * applies, the only spills that give the save a pair (bugs row 291). + */ +bool +txn_spill_identity_applies(HostIoIo *io, HostSharedRecord deltas, HostLsn point, bool *result, HostError *error) { - for (size_t k = frame->listed.f0.len; k > 0; k--) - { - const TxnSpillFile *file = &STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k - 1); - HostXid xid = {file->xid}; - HostStatus status; - TxnSpillNaming naming = {file->xid, file->kind, file->number, true, file->len, file->hash}; + txn_spill_identity_applies_frame_0_t frame = {.io = io, .deltas = deltas, .point = point, .result = result, + .error = error}; - if (txn_spill_replaced_by(frame->slotted.ptr, frame->slotted.len, file)) - continue; - if (!host_status(HOST_IO_IO_IO_DOORS(frame->io), xid, &status, frame->error)) - return false; - if (status == HOST_STATUS_ABORTED) - { - if (!txn_spill_named_push(frame, &frame->aborted, &file->xid, sizeof(uint64), _Alignof(uint64))) - return false; - continue; - } - if (!txn_spill_named_push(frame, &frame->files, &naming, sizeof(naming), _Alignof(TxnSpillNaming))) - return false; - } - return true; + std_landing(&frame, txn_spill_identity_applies_body_0, txn_spill_identity_applies_pad_0); + txn_spill_identity_applies_pad_0(&frame); + return frame.ok; } -/* txn_spill_named_read reads a pending file's length and its trailer's xxh3 from the file. */ -static bool -txn_spill_named_read(txn_spill_named_frame_0_t *frame, const TxnSpillNaming *naming, uint64 *len, uint64 *hash) +/* txn_spill_identity_folded_frame_0_t is identity_folded's state across its pad: the files listed, the xids seen and folded, and the files taken out, while live. */ +typedef struct txn_spill_identity_folded_frame_0_t { - TxnRunsNames names; - ChunksErrorError failed; + HostIoIo *io; + HostSharedRecord deltas; + HostLsn point; + HostError *error; + bool ok; + HostStorage storage; + txn_TupleVecFileU64_t listed; + bool listed_live; + StdVec seen; + bool seen_live; + StdVec folded; + bool folded_live; + StdVec gone; + bool gone_live; + bool *took; +} txn_spill_identity_folded_frame_0_t; - if (!txn_spill_names(frame->home, frame->state, frame->storage, naming->kind, naming->xid, &names, frame->error)) - return false; - txn_runs_spill_name(&names, naming->number, &frame->name); - frame->name_live = true; - std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, - &frame->path); - frame->path_live = true; - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - if (!chunks_file_open_rw(HOST_IO_IO_IO_DOORS(frame->io), frame->path.bytes.ptr, frame->path.bytes.len, false, - &frame->file, &failed)) - { - *frame->error = txn_base_storage(&failed); - return false; - } - frame->file_live = true; - if (!host_size(HOST_IO_IO_READ_DOORS(frame->io), &frame->file, len, frame->error) || - !txn_runs_spill_tail_hash(frame->io, &frame->file, *len, hash, frame->error)) - return false; - frame->file_live = false; - host_File_drop(&frame->file); - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); - return true; +/* txn_spill_identity_folded_pad_0 is identity_folded's pad. */ +static void +txn_spill_identity_folded_pad_0(void *frame_) +{ + txn_spill_identity_folded_frame_0_t *frame = frame_; + + if (frame->gone_live) + std_vec_free_Checked(&frame->gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (frame->folded_live) + std_vec_free_Checked(&frame->folded, sizeof(uint64), _Alignof(uint64)); + if (frame->seen_live) + std_vec_free_Checked(&frame->seen, sizeof(uint64), _Alignof(uint64)); + if (frame->listed_live) + std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + frame->gone_live = frame->folded_live = frame->seen_live = frame->listed_live = false; } -/* txn_spill_named_body_0 is named's body. */ +/* txn_spill_identity_folded_body_0 is identity_folded's body. */ static void -txn_spill_named_body_0(void *frame_) +txn_spill_identity_folded_body_0(void *frame_) { - txn_spill_named_frame_0_t *frame = frame_; + txn_spill_identity_folded_frame_0_t *frame = frame_; + TxnStatePersistence persistence; - std_vec_new(&frame->files, sizeof(TxnSpillNaming), _Alignof(TxnSpillNaming)); - frame->files_live = true; - std_vec_new(&frame->aborted, sizeof(uint64), _Alignof(uint64)); - frame->aborted_live = true; - if (!txn_spill_consistent(frame->home, frame->io, frame->state, frame->storage, &frame->slotted, - &frame->slotted_live, &frame->listed, &frame->listed_live, frame->error)) + *frame->took = false; + if (!txn_spill_Identity_Home_described(frame->deltas, &frame->storage, &persistence)) { - txn_spill_named_pad_0(frame); + frame->ok = true; return; } - if (!txn_spill_named_listed(frame)) - { - txn_spill_named_pad_0(frame); + if (!txn_spill_listed(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, frame->deltas, &frame->storage, &frame->listed, + frame->error)) return; - } - if (frame->aborted.len > 0) - { - if (!txn_spill_removed(frame->home, frame->io, frame->state, frame->storage, txn_spill_xids_closure_0, - &frame->aborted, &frame->gone, frame->error)) - { - txn_spill_named_pad_0(frame); - return; - } - frame->gone_live = true; - } - for (size_t k = 0; k < frame->slotted.len; k++) + frame->listed_live = true; + std_vec_new(&frame->seen, sizeof(uint64), _Alignof(uint64)); + frame->seen_live = true; + std_vec_new(&frame->folded, sizeof(uint64), _Alignof(uint64)); + frame->folded_live = true; + for (size_t k = 0; k < frame->listed.f0.len; k++) { - const txn_TupleU64KindU32OptionU32_t *slot = &STD_VEC_AT(&frame->slotted, txn_TupleU64KindU32OptionU32_t, k); - TxnSpillNaming naming = {slot->f0, slot->f1, slot->f2}; - bool named = false; - - for (size_t j = 0; j < frame->files.len && !named; j++) - { - const TxnSpillNaming *found = &STD_VEC_AT(&frame->files, TxnSpillNaming, j); + uint64 xid = STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k).xid; + bool applied; - named = found->xid == slot->f0 && found->kind == slot->f1 && found->number == slot->f2; - } - if (!named && !txn_spill_named_push(frame, &frame->files, &naming, sizeof(naming), _Alignof(TxnSpillNaming))) + if (txn_spill_has_xid(&frame->seen, xid)) + continue; + if (!std_vec_try_reserve(&frame->seen, 1, sizeof(uint64), _Alignof(uint64)) || + !std_vec_try_reserve(&frame->folded, 1, sizeof(uint64), _Alignof(uint64))) { - txn_spill_named_pad_0(frame); + *frame->error = host_Error_out_of_memory(); return; } + std_vec_push_Checked(&frame->seen, &xid, sizeof(uint64), _Alignof(uint64)); + if (!txn_spill_identity_applied(frame->io, xid, frame->point, &applied, frame->error)) + return; + if (applied) + std_vec_push_Checked(&frame->folded, &xid, sizeof(uint64), _Alignof(uint64)); } - txn_runs_spill_directory(frame->storage->database, frame->home->own(frame->state), &frame->dir); - frame->dir_live = true; - std_vec_new(frame->result, sizeof(RepoRunFile), _Alignof(RepoRunFile)); - frame->result_live = true; - if (!std_vec_try_reserve(frame->result, frame->files.len, sizeof(RepoRunFile), _Alignof(RepoRunFile))) - { - *frame->error = host_Error_out_of_memory(); - txn_spill_named_pad_0(frame); - return; - } - for (size_t k = 0; k < frame->files.len; k++) + if (frame->folded.len > 0) { - const TxnSpillNaming *naming = &STD_VEC_AT(&frame->files, TxnSpillNaming, k); - RepoRunFile run = {.kind = (uint8) naming->kind, .tablespace = frame->storage->tablespace, - .relation = frame->storage->relation, .len = naming->len, .hash = naming->hash, .has_spilled = true, - .spilled = {naming->xid, naming->number}}; - - if (!naming->has_read && !txn_spill_named_read(frame, naming, &run.len, &run.hash)) - { - txn_spill_named_pad_0(frame); + if (!txn_spill_removed(&TXN_SPILL_IDENTITY_HOME_OPS, frame->io, frame->deltas, &frame->storage, + txn_spill_xids_closure_0, &frame->folded, &frame->gone, frame->error)) return; - } - std_vec_push_Checked(frame->result, &run, sizeof(run), _Alignof(RepoRunFile)); + frame->gone_live = true; + *frame->took = frame->gone.len > 0; } - frame->result_live = false; frame->ok = true; - txn_spill_named_pad_0(frame); } /* - * txn_spill_named returns the manifest's entries of a storage's spill files - * a checkpoint names, each synced at its write: the listed oldest-linked - * first, then those processes' slots name. It reads the slots and the - * manifest as a reader does, so a merge pending its file names that file and - * not the sources it replaces. A listed spill of a transaction the commit - * log calls aborted leaves the manifest unnamed. + * txn_spill_identity_folded takes out of a working state's identity spills' + * manifest the files of every committed transaction whose commit a save's + * fold of the identity tree at a point applied (bugs row 291), answering + * whether it took one out; a later checkpoint unlinks the files it took out. */ bool -txn_spill_named(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, const HostStorage *storage, - StdVec *result, HostError *error) +txn_spill_identity_folded(HostIoIo *io, HostSharedRecord deltas, HostLsn point, bool *took, HostError *error) { - txn_spill_named_frame_0_t frame = {.home = home, .io = io, .state = state, .storage = storage, .result = result, + txn_spill_identity_folded_frame_0_t frame = {.io = io, .deltas = deltas, .point = point, .took = took, .error = error}; - std_landing(&frame, txn_spill_named_body_0, txn_spill_named_pad_0); + std_landing(&frame, txn_spill_identity_folded_body_0, txn_spill_identity_folded_pad_0); + txn_spill_identity_folded_pad_0(&frame); return frame.ok; } -/* txn_spill_settled_from_frame_0_t is settled_from's state across its pad: the spill's path, bytes, and decoding, its twins in the memtable, the hold and a key's chain, the tuple-id file's bytes and rows, and its tids, while live. */ -typedef struct txn_spill_settled_from_frame_0_t +/* TxnSpillRead is Read: a backend's read of a spill file, its path, length, and names, its index frame's place and entries, and the chunk it read last, by its entry. */ +typedef struct TxnSpillRead { - const TxnSpillHomeOps *home; + StdPathBuf path; + uint64 len; + TxnRunsNames names; + uint64 index_at; + /* entries is an Index, a Vec<(Vec, u64)>. */ + StdVec entries; + bool has_chunk; + size_t chunk_at; + /* chunk is a Chunk, a Vec<(Vec, Vec>)>. */ + StdVec chunk; +} TxnSpillRead; + +/* txn_spill_Read_drop is Read's drop glue: its path, entries, and chunk. */ +static void +txn_spill_Read_drop(void *elem) +{ + TxnSpillRead *self = elem; + + if (self->has_chunk) + txn_spill_spilled_drop(&self->chunk, 0); + txn_runs_Index_drop(&self->entries); + std_vec_free_Checked(&self->path.bytes, 1, 1); +} + +/* TXN_SPILL_KEPT is KEPT, the most spill files a backend keeps its reads of. */ +#define TXN_SPILL_KEPT 8 + +/* txn_spill_VecRead_drop is Vec's drop glue. */ +static void +txn_spill_VecRead_drop(void *value) +{ + std_vec_drop_Checked(value, sizeof(TxnSpillRead), _Alignof(TxnSpillRead), txn_spill_Read_drop); +} + +/* taken at Vec and IoMutex>, which spill.c alone uses. */ +#define HOST_LOCK_SCOPE static +#define HOST_LOCK_DECLARE +#define HOST_LOCK_DEFINE +#define HOST_LOCK_TAKEN_TS VecRead +#define HOST_LOCK_TAKEN_T StdVec +#define HOST_LOCK_TAKEN_DEFAULT(place) std_vec_new((place), sizeof(TxnSpillRead), _Alignof(TxnSpillRead)) +#define HOST_LOCK_TAKEN_DROP(place) txn_spill_VecRead_drop(place) +#define HOST_LOCK_TAKEN_TYPE TXN_SPILL_READS_LOCK_TYPE +#define HOST_LOCK_IO_MUTEX +#define HOST_LOCK_TS VecRead +#define HOST_LOCK_T StdVec +#define HOST_LOCK_DEFAULT(place) std_vec_new((place), sizeof(TxnSpillRead), _Alignof(TxnSpillRead)) +#include "doltgres/host/lock_template.h" + +/* txn_spill_IoMutexVecRead_drop is IoMutex>' drop glue: its value's. */ +static void +txn_spill_IoMutexVecRead_drop(void *value) +{ + HostLockIoMutexVecRead *self = value; + + txn_spill_VecRead_drop(&self->value); +} + +/* TXN_SPILL_READS_TYPE is IoMutex>' size, Default, and drop. */ +static const StdDefaultType TXN_SPILL_READS_TYPE = {sizeof(HostLockIoMutexVecRead), _Alignof(HostLockIoMutexVecRead), +host_lock_IoMutexVecRead_Default_default, txn_spill_IoMutexVecRead_drop}; + +/* + * TXN_SPILL_READS is READS: the backend's reads of the spill files it read + * last, the latest first, so a scan's steps read each chunk once; its + * BackendLocal header and the process's value, which follows it. + */ +static struct +{ + HostLocalBackendLocal local; + HostLockIoMutexVecRead process; +} TXN_SPILL_READS = {HOST_LOCAL_BACKEND_LOCAL_NEW}; + +/* txn_spill_chunk_frame_0_t is Read's chunk's state across its pad: the chunk it replaces, the file, and the chunk read, while live. */ +typedef struct txn_spill_chunk_frame_0_t +{ + TxnSpillRead *self; HostIoIo *io; - HostSharedRecord record; - const TxnSpillPended *pended; + size_t entry; HostError *error; bool ok; - StdPathBuf path; - bool path_live; - StdString name; - bool name_live; - StdVec bytes; - bool bytes_live; - TxnRunsDecoded decoded; - bool decoded_live; - /* twins is a Vec<(Vec, Vec)>. */ - StdVec twins; - bool twins_live; - HostDeltaHold hold; - bool hold_live; - /* chain is a Vec. */ - StdVec chain; - bool chain_live; - /* handles is a Vec of a key's twins. */ - StdVec handles; - bool handles_live; - StdVec tuple_bytes; - bool tuple_bytes_live; - /* rows is decode_tuples' Vec<(u64, &[u8])>. */ - StdVec rows; - bool rows_live; - /* tids is a Vec. */ - StdVec tids; - bool tids_live; -} txn_spill_settled_from_frame_0_t; + StdVec old; + bool old_live; + HostFile file; + bool file_live; + StdVec read; + bool read_live; +} txn_spill_chunk_frame_0_t; -/* txn_spill_settled_from_pad_0 is settled_from's pad: what it holds, while live. */ +/* txn_spill_chunk_pad_0 is Read's chunk's pad: what it holds, while live. */ static void -txn_spill_settled_from_pad_0(void *frame_) +txn_spill_chunk_pad_0(void *frame_) { - txn_spill_settled_from_frame_0_t *frame = frame_; + txn_spill_chunk_frame_0_t *frame = frame_; - if (frame->tids_live) - { - frame->tids_live = false; - std_vec_free_Checked(&frame->tids, sizeof(uint64), _Alignof(uint64)); - } - if (frame->rows_live) - { - frame->rows_live = false; - std_vec_free_Checked(&frame->rows, sizeof(txn_TupleU64RefSliceU8_t), _Alignof(txn_TupleU64RefSliceU8_t)); - } - if (frame->tuple_bytes_live) + if (frame->read_live) { - frame->tuple_bytes_live = false; - std_vec_free_Checked(&frame->tuple_bytes, 1, 1); + frame->read_live = false; + txn_spill_spilled_drop(&frame->read, 0); } - if (frame->handles_live) + if (frame->file_live) { - frame->handles_live = false; - std_vec_free_Checked(&frame->handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + frame->file_live = false; + host_File_drop(&frame->file); } - if (frame->chain_live) + if (frame->old_live) { - frame->chain_live = false; - txn_memtable_version_versions_drop(&frame->chain, 0); + frame->old_live = false; + txn_spill_spilled_drop(&frame->old, 0); } - if (frame->hold_live) +} + +/* txn_spill_chunk_body_0 is Read's chunk's body. */ +static void +txn_spill_chunk_body_0(void *frame_) +{ + txn_spill_chunk_frame_0_t *frame = frame_; + TxnSpillRead *self = frame->self; + ChunksErrorError failed; + uint64 start; + uint64 end = self->index_at; + size_t next; + + if (self->has_chunk && self->chunk_at == frame->entry) { - frame->hold_live = false; - host_delta_Hold_drop(&frame->hold); + frame->ok = true; + return; } - if (frame->twins_live) + frame->old = self->chunk; + frame->old_live = self->has_chunk; + self->has_chunk = false; + if (!chunks_file_open_rw(HOST_IO_IO_IO_DOORS(frame->io), self->path.bytes.ptr, self->path.bytes.len, false, + &frame->file, &failed)) { - frame->twins_live = false; - txn_spill_taken_drop(&frame->twins); + *frame->error = txn_base_storage(&failed); + txn_spill_chunk_pad_0(frame); + return; } - if (frame->decoded_live) + frame->file_live = true; + std_index_check(frame->entry, self->entries.len, RUST_FILE, 1679, 29); + start = STD_VEC_AT(&self->entries, txn_TupleVecU8U64_t, frame->entry).f1; + next = STD_ADD(usize, frame->entry, 1, 1680, 44); + if (next < self->entries.len) + end = STD_VEC_AT(&self->entries, txn_TupleVecU8U64_t, next).f1; + if (!txn_runs_spill_chunk(frame->io, &frame->file, &self->names, start, end, &frame->read, frame->error)) { - frame->decoded_live = false; - txn_runs_Decoded_drop(&frame->decoded); + txn_spill_chunk_pad_0(frame); + return; } - if (frame->bytes_live) + frame->read_live = true; + frame->file_live = false; + host_File_drop(&frame->file); + if (frame->old_live) { - frame->bytes_live = false; - std_vec_free_Checked(&frame->bytes, 1, 1); + frame->old_live = false; + txn_spill_spilled_drop(&frame->old, 0); } - if (frame->name_live) + self->chunk = frame->read; + self->chunk_at = frame->entry; + self->has_chunk = true; + frame->read_live = false; + frame->ok = true; +} + +/* txn_spill_Read_chunk reads the keys of the chunk an index entry starts into the read, once. */ +static bool +txn_spill_Read_chunk(TxnSpillRead *self, HostIoIo *io, size_t entry, HostError *error) +{ + txn_spill_chunk_frame_0_t frame = {.self = self, .io = io, .entry = entry, .error = error}; + + std_landing(&frame, txn_spill_chunk_body_0, txn_spill_chunk_pad_0); + return frame.ok; +} + +/* txn_spill_Read_entries_before returns how many of the read's index entries start at or below a key, or below it when strictly. */ +static size_t +txn_spill_Read_entries_before(const TxnSpillRead *self, const uint8 *key, size_t key_len, bool strictly) +{ + size_t low = 0; + size_t high = self->entries.len; + + while (low < high) { - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); + size_t middle = low + (high - low) / 2; + const StdVec *first = &STD_VEC_AT(&self->entries, txn_TupleVecU8U64_t, middle).f0; + StdOrdering order = std_bytes_Ord_cmp(first->ptr, first->len, key, key_len); + + if (order == STD_ORDERING_LESS || (!strictly && order == STD_ORDERING_EQUAL)) + low = middle + 1; + else + high = middle; } - if (frame->path_live) + return low; +} + +/* txn_spill_vec_u8_clone is Vec's Clone, into a place. */ +static void +txn_spill_vec_u8_clone(const void *elem, void *result) +{ + const StdVec *record = elem; + + std_slice_to_vec_Checked(result, record->ptr, record->len, 1, 1); +} + +/* txn_spill_versions_clone clones a Vec> of records. */ +static void +txn_spill_versions_clone(const StdVec *records, StdVec *result) +{ + std_slice_to_vec_clone(result, records->ptr, records->len, sizeof(StdVec), _Alignof(StdVec), + txn_spill_vec_u8_clone, txn_spill_vec_u8_drop); +} + +/* txn_spill_clone_frame_0_t is Spilled's clone's state across its pad: the key cloned, while live. */ +typedef struct txn_spill_clone_frame_0_t +{ + const txn_TupleVecU8VecVecU8_t *self; + txn_TupleVecU8VecVecU8_t *result; + bool key_live; +} txn_spill_clone_frame_0_t; + +/* txn_spill_clone_pad_0 is Spilled's clone's pad: the key cloned, while live. */ +static void +txn_spill_clone_pad_0(void *frame_) +{ + txn_spill_clone_frame_0_t *frame = frame_; + + if (frame->key_live) { - frame->path_live = false; - std_vec_free_Checked(&frame->path.bytes, 1, 1); + frame->key_live = false; + std_vec_free_Checked(&frame->result->f0, 1, 1); } } -/* txn_spill_settled_from_read reads a spill's file of the pending names whole into a buffer, with its xxh3, its filter's bits, and its length. */ +/* txn_spill_clone_body_0 is Spilled's clone's body: the key, then its records. */ +static void +txn_spill_clone_body_0(void *frame_) +{ + txn_spill_clone_frame_0_t *frame = frame_; + + std_slice_to_vec_Checked(&frame->result->f0, frame->self->f0.ptr, frame->self->f0.len, 1, 1); + frame->key_live = true; + txn_spill_versions_clone(&frame->self->f1, &frame->result->f1); + frame->key_live = false; +} + +/* txn_spill_Spilled_clone is Spilled's Clone: a key of a spill's file with its records copied, which a scan's step keeps. */ +static void +txn_spill_Spilled_clone(const txn_TupleVecU8VecVecU8_t *self, txn_TupleVecU8VecVecU8_t *result) +{ + txn_spill_clone_frame_0_t frame = {.self = self, .result = result}; + + std_landing(&frame, txn_spill_clone_body_0, txn_spill_clone_pad_0); +} + +/* txn_spill_Read_looked_up returns a key's versions' records, newest first, or none for a key the file doesn't hold. */ static bool -txn_spill_settled_from_read(txn_spill_settled_from_frame_0_t *frame, const TxnRunsNames *names, StdVec *bytes, - bool *live, bool bits, TxnSpillFile *result) +txn_spill_Read_looked_up(TxnSpillRead *self, HostIoIo *io, const uint8 *key, size_t key_len, StdVec *result, + HostError *error) { - ChunksErrorError failed; + size_t entry = txn_spill_Read_entries_before(self, key, key_len, false); + size_t low = 0; + size_t high; - txn_runs_spill_name(names, frame->pended->number, &frame->name); - frame->name_live = true; - if (frame->path_live) - std_PathBuf_set_file_name(&frame->path, (const char *) frame->name.ptr, frame->name.len); - else + std_vec_new(result, sizeof(StdVec), _Alignof(StdVec)); + if (entry == 0) + return true; + if (!txn_spill_Read_chunk(self, io, entry - 1, error)) + return false; + high = self->chunk.len; + while (low < high) { - txn_runs_spill_directory(frame->pended->storage.database, frame->home->own(frame->record), &frame->path); - frame->path_live = true; - { - StdPathBuf joined; + size_t middle = low + (high - low) / 2; + const txn_TupleVecU8VecVecU8_t *found = &STD_VEC_AT(&self->chunk, txn_TupleVecU8VecVecU8_t, middle); + StdOrdering order = std_bytes_Ord_cmp(found->f0.ptr, found->f0.len, key, key_len); - std_path_join(frame->path.bytes.ptr, frame->path.bytes.len, (const char *) frame->name.ptr, - frame->name.len, &joined); - std_vec_free_Checked(&frame->path.bytes, 1, 1); - frame->path = joined; + if (order == STD_ORDERING_EQUAL) + { + txn_spill_versions_clone(&found->f1, result); + return true; } + if (order == STD_ORDERING_LESS) + low = middle + 1; + else + high = middle; } - frame->name_live = false; - std_vec_free_Checked(&frame->name, 1, 1); - if (!chunks_file_read_file(HOST_IO_IO_READ_DOORS(frame->io), frame->path.bytes.ptr, frame->path.bytes.len, bytes, - &failed)) - { - *frame->error = txn_base_storage(&failed); - return false; - } - *live = true; - result->xid = frame->pended->xid; - result->kind = names->kind; - result->number = frame->pended->number; - result->len = (uint64) bytes->len; - return txn_runs_spill_hash(bytes->ptr, bytes->len, &result->hash, frame->error) && - (!bits || txn_runs_spill_bits(bytes->ptr, bytes->len, &result->bits, frame->error)); + return true; } -/* txn_spill_settled_from_twins gathers each version of a key the file holds whose record the memtable's chain holds too, by its bytes, and takes them out. */ +/* txn_spill_Read_next returns the file's first key past a place the way a scan goes, with its versions' records, or nothing past its last. */ static bool -txn_spill_settled_from_twins(txn_spill_settled_from_frame_0_t *frame, TxnWrittenOf of) +txn_spill_Read_next(TxnSpillRead *self, HostIoIo *io, bool has_after, const uint8 *after, size_t after_len, + TxnMemtableTableDirection way, bool *some, txn_TupleVecU8VecVecU8_t *result, HostError *error) { - std_vec_new(&frame->twins, sizeof(txn_TupleVecU8VecHandle_t), _Alignof(txn_TupleVecU8VecHandle_t)); - frame->twins_live = true; - if (!host_delta_hold(frame->io, &frame->hold, frame->error)) - return false; - frame->hold_live = true; - for (size_t k = 0; k < frame->decoded.keys.len; k++) + size_t entries = self->entries.len; + size_t entry; + + *some = false; + if (!has_after) + entry = way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD ? 0 : std_saturating_sub_usize(entries, 1); + else if (way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD) + entry = std_saturating_sub_usize(txn_spill_Read_entries_before(self, after, after_len, false), 1); + else { - const TxnRunsKey *key = &STD_VEC_AT(&frame->decoded.keys, TxnRunsKey, k); - txn_TupleVecU8VecHandle_t twin; + entry = txn_spill_Read_entries_before(self, after, after_len, true); + if (entry == 0) + return true; + entry--; + } + while (entry < entries) + { + const txn_TupleVecU8VecVecU8_t *found = NULL; - if (!txn_spill_Home_chain(frame->home, frame->io, &frame->hold, frame->record, of, key->key.ptr, key->key.len, - &frame->chain, frame->error)) + if (!txn_spill_Read_chunk(self, io, entry, error)) return false; - frame->chain_live = true; - std_vec_new(&frame->handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); - frame->handles_live = true; - for (size_t c = 0; c < frame->chain.len; c++) + for (size_t k = 0; k < self->chunk.len && found == NULL; k++) { - const TxnMemtableVersionVersion *version = &STD_VEC_AT(&frame->chain, TxnMemtableVersionVersion, c); - TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); - StdBytes bytes = txn_memtable_version_Record_bytes(&record); - bool twinned = false; - - for (size_t v = 0; v < key->versions.len && !twinned; v++) - { - StdBytes kept = txn_memtable_version_Record_bytes(&STD_VEC_AT(&key->versions, - TxnMemtableVersionRecord, v)); + size_t at = way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD ? k : self->chunk.len - 1 - k; + const txn_TupleVecU8VecVecU8_t *key = &STD_VEC_AT(&self->chunk, txn_TupleVecU8VecVecU8_t, at); + StdOrdering order = has_after ? std_bytes_Ord_cmp(key->f0.ptr, key->f0.len, after, after_len) : + STD_ORDERING_EQUAL; - twinned = kept.len == bytes.len && memcmp(kept.ptr, bytes.ptr, bytes.len) == 0; - } - if (twinned) - std_vec_push_Checked(&frame->handles, &version->handle, sizeof(version->handle), - _Alignof(TxnMemtableVersionHandle)); - } - frame->chain_live = false; - txn_memtable_version_versions_drop(&frame->chain, 0); - if (frame->handles.len == 0) - { - frame->handles_live = false; - std_vec_free_Checked(&frame->handles, sizeof(TxnMemtableVersionHandle), - _Alignof(TxnMemtableVersionHandle)); - continue; + if (!has_after || (way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD ? order == STD_ORDERING_GREATER : + order == STD_ORDERING_LESS)) + found = key; } - if (!std_vec_try_reserve(&frame->twins, 1, sizeof(twin), _Alignof(txn_TupleVecU8VecHandle_t))) + if (found != NULL) { - *frame->error = host_Error_out_of_memory(); - return false; + txn_spill_Spilled_clone(found, result); + *some = true; + return true; } - std_slice_to_vec_Checked(&twin.f0, key->key.ptr, key->key.len, 1, 1); - twin.f1 = frame->handles; - frame->handles_live = false; - std_vec_push_Checked(&frame->twins, &twin, sizeof(twin), _Alignof(txn_TupleVecU8VecHandle_t)); + if (way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD) + entry = STD_ADD(usize, entry, 1, 1728, 39); + else if (entry == 0) + return true; + else + entry--; } - frame->hold_live = false; - host_delta_Hold_drop(&frame->hold); - return txn_spill_removed_keys(frame->home, frame->io, frame->record, of, &frame->twins, frame->error); + return true; } -/* txn_spill_settled_from_body_0 is settled_from's body. */ -static void -txn_spill_settled_from_body_0(void *frame_) +/* txn_spill_reading_frame_0_t is reading's state across its pad: the backend's reads and their guard, the read it runs, and the file it opens, while live. */ +typedef struct txn_spill_reading_frame_0_t { - txn_spill_settled_from_frame_0_t *frame = frame_; - const TxnSpillPended *pended = frame->pended; - TxnRunsNames names; - TxnSpillFile listing[2]; - size_t listing_len = 1; - TxnWrittenOf of; - txn_TupleU64KindU32OptionU32_t replacing = {pended->xid, pended->kind, pended->number, true, pended->sources}; - - if (!txn_spill_names(frame->home, frame->record, &pended->storage, pended->kind, pended->xid, &names, - frame->error) || - !txn_spill_settled_from_read(frame, &names, &frame->bytes, &frame->bytes_live, true, &listing[0]) || - !txn_runs_decode(frame->bytes.ptr, frame->bytes.len, &names, listing[0].hash, &frame->decoded, frame->error)) - { - txn_spill_settled_from_pad_0(frame); - return; + HostIoIo *io; + const StdPathBuf *path; + const TxnRunsNames *names; + bool has_len; + uint64 len; + bool (*with) (void *captures, HostIoIo *io, TxnSpillRead *read, HostError *error); + void *captures; + HostError *error; + bool ok; + HostLocalLocal reads; + bool reads_live; + HostLockLockedRwLockWriteGuard guard; + bool guard_live; + TxnSpillRead read; + bool read_live; + HostFile file; + bool file_live; +} txn_spill_reading_frame_0_t; + +/* txn_spill_reading_pad_0 is reading's pad: what it holds, while live. */ +static void +txn_spill_reading_pad_0(void *frame_) +{ + txn_spill_reading_frame_0_t *frame = frame_; + + if (frame->file_live) + { + frame->file_live = false; + host_File_drop(&frame->file); } - frame->decoded_live = true; - switch (pended->kind) + if (frame->read_live) { - case TXN_RUNS_KIND_SPILLED_ROWS: - case TXN_RUNS_KIND_SPILLED_ENTRIES: - of = TXN_WRITTEN_OF_ROWS; - break; - case TXN_RUNS_KIND_SPILLED_VALUES: - of = TXN_WRITTEN_OF_VALUES; - break; - case TXN_RUNS_KIND_SPILLED_CHUNKS: - of = TXN_WRITTEN_OF_CHUNKS; - break; - default: - *frame->error = txn_spill_internal("a process's slot names a spill of no tree"); - txn_spill_settled_from_pad_0(frame); - return; + frame->read_live = false; + txn_spill_Read_drop(&frame->read); } - if (!pended->has_sources && !txn_spill_settled_from_twins(frame, of)) + if (frame->guard_live) { - txn_spill_settled_from_pad_0(frame); - return; + frame->guard_live = false; + host_lock_LockedRwLockWriteGuard_drop(&frame->guard); } - if (pended->tupled) + if (frame->reads_live) { - TxnRunsNames tuple_names; - - if (!txn_spill_names(frame->home, frame->record, &pended->storage, TXN_RUNS_KIND_SPILLED_TUPLES, pended->xid, - &tuple_names, frame->error) || - !txn_spill_settled_from_read(frame, &tuple_names, &frame->tuple_bytes, &frame->tuple_bytes_live, false, - &listing[1])) - { - txn_spill_settled_from_pad_0(frame); - return; - } - if (!pended->has_sources) - { - if (!txn_runs_decode_tuples(frame->tuple_bytes.ptr, frame->tuple_bytes.len, &tuple_names, listing[1].hash, - &frame->rows, frame->error)) - { - txn_spill_settled_from_pad_0(frame); - return; - } - frame->rows_live = true; - std_vec_with_capacity_Checked(&frame->tids, frame->rows.len, sizeof(uint64), _Alignof(uint64)); - frame->tids_live = true; - for (size_t k = 0; k < frame->rows.len; k++) - std_vec_push_Checked(&frame->tids, &STD_VEC_AT(&frame->rows, txn_TupleU64RefSliceU8_t, k).f0, - sizeof(uint64), _Alignof(uint64)); - if (!frame->home->untupled(frame->io, frame->record, frame->tids.ptr, frame->tids.len, frame->error)) - { - txn_spill_settled_from_pad_0(frame); - return; - } - } - if (!txn_runs_spill_bits(frame->tuple_bytes.ptr, frame->tuple_bytes.len, &listing[1].bits, frame->error)) - { - txn_spill_settled_from_pad_0(frame); - return; - } - listing_len = 2; + frame->reads_live = false; + host_local_Local_drop(&frame->reads, &TXN_SPILL_READS_TYPE); } - frame->ok = txn_spill_linked(frame->home, frame->io, frame->record, &pended->storage, listing, listing_len, - pended->has_sources, replacing, frame->error) && - txn_spill_settled(frame->home, frame->io, &frame->record, frame->error); - txn_spill_settled_from_pad_0(frame); } -/* txn_spill_settled_from lists a pending spill in its home's manifest as settle_pending does. */ -static bool -txn_spill_settled_from(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, - const TxnSpillPended *pended, HostError *error) +/* txn_spill_reading_lock locks the backend's reads, answering them. */ +static StdVec * +txn_spill_reading_lock(txn_spill_reading_frame_0_t *frame) { - txn_spill_settled_from_frame_0_t frame = {.home = home, .io = io, .record = record, .pended = pended, - .error = error}; - - std_landing(&frame, txn_spill_settled_from_body_0, txn_spill_settled_from_pad_0); - return frame.ok; + frame->guard = host_lock_IoMutex_lock((HostLockIoMutex *) host_local_Local_Deref_deref(&frame->reads, + &TXN_SPILL_READS_TYPE), + &TXN_SPILL_READS_LOCK_TYPE, &frame->io->held); + frame->guard_live = true; + return host_lock_LockedRwLockWriteGuard_DerefMut_deref_mut(&frame->guard); } -/* txn_spill_settle_pending_captures_t is settle_pending's step's captures: the backend's I/O and its pending spill. */ -typedef struct txn_spill_settle_pending_captures_t -{ - HostIoIo *io; - const TxnSpillPended *pended; -} txn_spill_settle_pending_captures_t; - -/* txn_spill_settle_pending_closure_0 is settle_pending's step: the pending spill listed in its home's manifest. */ +/* txn_spill_reading_opened opens the file the backend keeps no read of and reads its index into the frame's read. */ static bool -txn_spill_settle_pending_closure_0(void *captures, const TxnSpillHomeOps *home, const HostSharedHeld *held, - HostError *error) -{ - txn_spill_settle_pending_captures_t *c = captures; - - return txn_spill_settled_from(home, c->io, host_shared_Held_record(held), c->pended, error); -} - -/* - * txn_spill_settle_pending lists the spill the process's slot names, read - * back from its file, and clears the slot; a slot whose storage has no state - * any more is cleared alone. It first takes out of the memtable each version - * of a key the file holds whose record is one the file holds, by its bytes, - * and drops the tids of the spill's tuple-id file, then lists the spill's - * tuple-id file after its row file, and a merge's files in place of those it - * replaces. A bootstrap process has no slot. - */ -bool -txn_spill_settle_pending(HostIoIo *io, HostError *error) +txn_spill_reading_opened(txn_spill_reading_frame_0_t *frame) { - bool some; - TxnSpillPended pended; - txn_spill_settle_pending_captures_t captures = {io, &pended}; + ChunksErrorError failed; + TxnSpillRead *read = &frame->read; - if (!host_shared_ready()) - return true; - if (!txn_spill_pending(io, &some, &pended, error)) + if (!chunks_file_open_rw(HOST_IO_IO_IO_DOORS(frame->io), frame->path->bytes.ptr, frame->path->bytes.len, false, + &frame->file, &failed)) + { + *frame->error = txn_base_storage(&failed); return false; - if (!some) - return true; - if (!txn_spill_at_found(io, &pended.storage, txn_spill_settle_pending_closure_0, &captures, &some, error)) + } + frame->file_live = true; + std_PathBuf_Clone_clone(frame->path, &read->path); + read->names = *frame->names; + read->has_chunk = false; + read->len = frame->len; + std_vec_new(&read->entries, sizeof(txn_TupleVecU8U64_t), _Alignof(txn_TupleVecU8U64_t)); + frame->read_live = true; + if ((!frame->has_len && !host_size(HOST_IO_IO_READ_DOORS(frame->io), &frame->file, &read->len, frame->error)) || + !txn_runs_spill_index(frame->io, &frame->file, read->len, &read->index_at, &read->entries, frame->error)) return false; - return some || txn_spill_settled(&TXN_SPILL_STATE_HOME_OPS, io, NULL, error); + frame->file_live = false; + host_File_drop(&frame->file); + return true; } -#ifdef DOLTGRES_TEST_TXN -#include "doltgres/simc/simc.h" -#include "doltgres/std/fmt.h" -#include "doltgres/std/test.h" -#include "doltgres/txn/memtable/testing.h" - -/* TXN_SPILL_TESTS_KIND_NAMES are Kind's variants' names, at their numbers, for its derived Debug. */ -static const char *const TXN_SPILL_TESTS_KIND_NAMES[] = {"Rows", "Free", "Values", "Chunks", "Status", "SpilledRows", -"SpilledValues", "SpilledTuples", "Entries", "SpilledEntries", "Identity", NULL, "SpilledChunks"}; - -/* txn_spill_tests_Kind_Debug_fmt is Kind's derived Debug, its variant's name, for the tests' asserts. */ -static bool -txn_spill_tests_Kind_Debug_fmt(const void *value, StdFormatter *f) +/* txn_spill_reading_body_0 is reading's body. */ +static void +txn_spill_reading_body_0(void *frame_) { - const char *name = TXN_SPILL_TESTS_KIND_NAMES[*(const TxnRunsKind *) value]; + txn_spill_reading_frame_0_t *frame = frame_; + StdVec *reads; + size_t at; - return std_fmt_write_str(f, name, strlen(name)); + frame->reads = host_local_BackendLocal_get(&TXN_SPILL_READS.local, &TXN_SPILL_READS_TYPE); + frame->reads_live = true; + reads = txn_spill_reading_lock(frame); + for (at = 0; at < reads->len; at++) + { + const TxnSpillRead *kept = &STD_VEC_AT(reads, TxnSpillRead, at); + + if (std_path_PartialEq_eq(kept->path.bytes.ptr, kept->path.bytes.len, frame->path->bytes.ptr, + frame->path->bytes.len) && (!frame->has_len || kept->len == frame->len)) + break; + } + if (at < reads->len) + { + std_vec_remove(reads, at, &frame->read, sizeof(frame->read), _Alignof(TxnSpillRead), RUST_FILE, + 1651, 29); + frame->read_live = true; + } + frame->guard_live = false; + host_lock_LockedRwLockWriteGuard_drop(&frame->guard); + if (!frame->read_live && !txn_spill_reading_opened(frame)) + { + txn_spill_reading_pad_0(frame); + return; + } + frame->ok = frame->with(frame->captures, frame->io, &frame->read, frame->error); + reads = txn_spill_reading_lock(frame); + std_vec_insert(reads, 0, &frame->read, sizeof(frame->read), _Alignof(TxnSpillRead), RUST_FILE, + 1667, 11); + frame->read_live = false; + std_vec_truncate(reads, TXN_SPILL_KEPT, sizeof(TxnSpillRead), txn_spill_Read_drop); + txn_spill_reading_pad_0(frame); } -/* txn_spill_tests_File_Debug_fmt is File's derived Debug, File { xid, kind, number, len, hash, bits }, for the tests' asserts. */ +/* + * txn_spill_reading runs a read of a spill's file of the length its + * manifest lists, or of its own for a pending file's, keeping the read first + * among the backend's and reading the index of a file it doesn't keep. + */ static bool -txn_spill_tests_File_Debug_fmt(const void *value, StdFormatter *f) +txn_spill_reading(HostIoIo *io, const StdPathBuf *path, const TxnRunsNames *names, bool has_len, uint64 len, + bool (*with) (void *captures, HostIoIo *io, TxnSpillRead *read, HostError *error), void *captures, + HostError *error) { - const TxnSpillFile *file = value; - StdDebugStruct fields = std_fmt_debug_struct(f, "File", 4); + txn_spill_reading_frame_0_t frame = {.io = io, .path = path, .names = names, .has_len = has_len, .len = len, + .with = with, .captures = captures, .error = error}; - std_DebugStruct_field(&fields, "xid", 3, &file->xid, std_u64_Debug_fmt); - std_DebugStruct_field(&fields, "kind", 4, &file->kind, txn_spill_tests_Kind_Debug_fmt); - std_DebugStruct_field(&fields, "number", 6, &file->number, std_u32_Debug_fmt); - std_DebugStruct_field(&fields, "len", 3, &file->len, std_u64_Debug_fmt); - std_DebugStruct_field(&fields, "hash", 4, &file->hash, std_u64_Debug_fmt); - std_DebugStruct_field(&fields, "bits", 4, &file->bits, std_u64_Debug_fmt); - return std_DebugStruct_finish(&fields); + std_landing(&frame, txn_spill_reading_body_0, txn_spill_reading_pad_0); + return frame.ok; } -/* txn_spill_tests_files_Debug_fmt is a Vec's Debug. */ +/* txn_spill_consistent reads a storage's slots and then lists its manifest, starting over while a change of the manifest moved its generation between the two, setting its caller's live flags of the two. */ static bool -txn_spill_tests_files_Debug_fmt(const void *value, StdFormatter *f) +txn_spill_consistent(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, const HostStorage *storage, + StdVec *slotted, bool *slotted_live, txn_TupleVecFileU64_t *listed, bool *listed_live, + HostError *error) { - const StdVec *files = value; - StdDebugList list = std_fmt_debug_list(f); + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; - for (size_t k = 0; k < files->len; k++) - std_DebugList_entry(&list, &STD_VEC_AT(files, TxnSpillFile, k), txn_spill_tests_File_Debug_fmt); - return std_DebugList_finish(&list); + for (;;) + { + uint64 before = host_shared_Record_atomic(state, home->changed, load); + + if (!txn_spill_pended(home, state, storage, slotted, error)) + return false; + *slotted_live = true; + if (!host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-slots-read", error) || + !txn_spill_listed(home, io, state, storage, listed, error)) + return false; + *listed_live = true; + if (before == listed->f1) + return true; + *listed_live = false; + std_vec_free_Checked(&listed->f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + *slotted_live = false; + std_vec_free_Checked(slotted, sizeof(txn_TupleU64KindU32OptionU32_t), _Alignof(txn_TupleU64KindU32OptionU32_t)); + } } -/* txn_spill_tests_files_are asserts at its place that a Vec is a list of files, and frees the Vec. */ -static void -txn_spill_tests_files_are(uint32 line, uint32 col, StdVec *files, const TxnSpillFile *wanted, size_t wanted_len) +/* TxnSpillReading is Reading: a spill file a reader reads, its xid, its kind, its number, and its length when its manifest lists it. */ +typedef struct TxnSpillReading { - StdVec wanted_vec = {(void *) wanted, wanted_len, wanted_len}; - bool eq = files->len == wanted_len; + uint64 xid; + TxnRunsKind kind; + uint32 number; + bool has_len; + uint64 len; +} TxnSpillReading; - for (size_t k = 0; eq && k < wanted_len; k++) - eq = txn_spill_File_PartialEq_eq(&STD_VEC_AT(files, TxnSpillFile, k), &wanted[k]); - STD_ASSERT_EQ(line, col, eq, files, txn_spill_tests_files_Debug_fmt, &wanted_vec, txn_spill_tests_files_Debug_fmt); - std_vec_free_Checked(files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); -} +/* txn_spill_files_frame_0_t is files' state across its pad: the slots, the listed files, the spills' directory, a file's path, and the readings, while live. */ +typedef struct txn_spill_files_frame_0_t +{ + const TxnSpillHomeOps *home; + HostIoIo *io; + HostSharedRecord state; + const HostStorage *storage; + TxnRunsKind kind; + bool has_key; + const uint8 *key; + size_t key_len; + StdVec *result; + HostError *error; + bool ok; + bool result_live; + StdVec slotted; + bool slotted_live; + txn_TupleVecFileU64_t listed; + bool listed_live; + StdPathBuf dir; + bool dir_live; + StdString name; + bool name_live; + StdPathBuf path; + bool path_live; + /* through is the point the newest base passed a map's spills at, or 0. */ + HostLsn through; + /* residuals is a Vec of the residuals read at that point. */ + StdVec residuals; + bool residuals_live; + /* passing is a Vec<(u64, u64)> of each transaction's passing point. */ + StdVec passing; + bool passing_live; +} txn_spill_files_frame_0_t; -/* txn_spill_tests_Slotted_Debug_fmt is Slotted's Debug, the tuple (xid, kind, number, sources). */ -static bool -txn_spill_tests_Slotted_Debug_fmt(const void *value, StdFormatter *f) +/* TxnSpillFilesResidual is a transaction's residual of a kind that files reads, with its fold's point and the greatest number. */ +typedef struct TxnSpillFilesResidual { - const txn_TupleU64KindU32OptionU32_t *slotted = value; - StdOptionRef sources = {slotted->has_f3, STD_ITEM(&slotted->f3, std_u32_Debug_fmt)}; - StdTupleRef tuple = STD_TUPLE(4, STD_ITEM(&slotted->f0, std_u64_Debug_fmt), - STD_ITEM(&slotted->f1, txn_spill_tests_Kind_Debug_fmt), - STD_ITEM(&slotted->f2, std_u32_Debug_fmt), - STD_ITEM(&sources, std_option_Debug_fmt)); + uint64 xid; + TxnRunsKind kind; + HostLsn from; + uint32 number; +} TxnSpillFilesResidual; - return std_tuple_Debug_fmt(&tuple, f); +/* txn_spill_files_pad_0 is files' pad: what it holds, while live. */ +static void +txn_spill_files_pad_0(void *frame_) +{ + txn_spill_files_frame_0_t *frame = frame_; + + if (frame->passing_live) + { + frame->passing_live = false; + std_vec_free_Checked(&frame->passing, sizeof(std_TupleU64U64_t), _Alignof(std_TupleU64U64_t)); + } + if (frame->residuals_live) + { + frame->residuals_live = false; + std_vec_free_Checked(&frame->residuals, sizeof(TxnSpillFilesResidual), _Alignof(TxnSpillFilesResidual)); + } + if (frame->path_live) + { + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + } + if (frame->name_live) + { + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + } + if (frame->dir_live) + { + frame->dir_live = false; + std_vec_free_Checked(&frame->dir.bytes, 1, 1); + } + if (frame->listed_live) + { + frame->listed_live = false; + std_vec_free_Checked(&frame->listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + } + if (frame->slotted_live) + { + frame->slotted_live = false; + std_vec_free_Checked(&frame->slotted, sizeof(txn_TupleU64KindU32OptionU32_t), + _Alignof(txn_TupleU64KindU32OptionU32_t)); + } + if (frame->result_live) + { + frame->result_live = false; + std_vec_free_Checked(frame->result, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); + } } -/* txn_spill_tests_slotteds_Debug_fmt is a Vec's Debug. */ +/* txn_spill_files_push adds a reading to the frame's result. */ static bool -txn_spill_tests_slotteds_Debug_fmt(const void *value, StdFormatter *f) +txn_spill_files_push(txn_spill_files_frame_0_t *frame, const TxnSpillReading *reading) { - const StdVec *slotteds = value; - StdDebugList list = std_fmt_debug_list(f); - - for (size_t k = 0; k < slotteds->len; k++) - std_DebugList_entry(&list, &STD_VEC_AT(slotteds, txn_TupleU64KindU32OptionU32_t, k), - txn_spill_tests_Slotted_Debug_fmt); - return std_DebugList_finish(&list); + if (!std_vec_try_reserve(frame->result, 1, sizeof(*reading), _Alignof(TxnSpillReading))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(frame->result, reading, sizeof(*reading), _Alignof(TxnSpillReading)); + return true; } -/* txn_spill_tests_Pended_Debug_fmt is Pended's Debug, the tuple (storage, xid, kind, number, tupled, sources). */ +/* txn_spill_files_hit reports whether the frame's key hits a listed file's filter. */ static bool -txn_spill_tests_Pended_Debug_fmt(const void *value, StdFormatter *f) +txn_spill_files_hit(txn_spill_files_frame_0_t *frame, const TxnSpillFile *file, bool *hit) { - const TxnSpillPended *pended = value; - StdOptionRef sources = {pended->has_sources, STD_ITEM(&pended->sources, std_u32_Debug_fmt)}; - StdTupleRef tuple = STD_TUPLE(6, STD_ITEM(&pended->storage, host_Storage_Debug_fmt), - STD_ITEM(&pended->xid, std_u64_Debug_fmt), - STD_ITEM(&pended->kind, txn_spill_tests_Kind_Debug_fmt), - STD_ITEM(&pended->number, std_u32_Debug_fmt), - STD_ITEM(&pended->tupled, std_bool_Debug_fmt), - STD_ITEM(&sources, std_option_Debug_fmt)); + TxnRunsNames names; - return std_tuple_Debug_fmt(&tuple, f); + if (!txn_spill_names(frame->home, frame->state, frame->storage, file->kind, file->xid, &names, frame->error)) + return false; + txn_runs_spill_name(&names, file->number, &frame->name); + frame->name_live = true; + std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, + &frame->path); + frame->path_live = true; + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + if (!txn_spill_hits(frame->io, &frame->path, file, frame->key, frame->key_len, hit, frame->error)) + return false; + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + return true; } -/* txn_spill_tests_file returns a test's spill file of rows by a transaction, of a number, which no file on disk holds. */ -static TxnSpillFile -txn_spill_tests_file(uint64 xid, uint32 number) +/* txn_spill_files_residuals gathers the residuals of the listed files read at the newest base's point: of each transaction and kind, the greatest number. */ +static bool +txn_spill_files_residuals(txn_spill_files_frame_0_t *frame) { - return (TxnSpillFile) + std_vec_new(&frame->residuals, sizeof(TxnSpillFilesResidual), _Alignof(TxnSpillFilesResidual)); + frame->residuals_live = true; + for (size_t k = 0; k < frame->listed.f0.len; k++) { - .xid = xid, .kind = TXN_RUNS_KIND_SPILLED_ROWS, .number = number, .len = 100 + (uint64) number, - .hash = xid ^ 0xfeed, .bits = 64 - }; -} + const TxnSpillFile *file = &STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k); + TxnSpillFilesResidual residual = {file->xid, file->kind, file->from, file->number}; + bool found = false; -/* txn_spill_tests_files_of returns the files a state's storage's manifest lists, newest first. */ -static StdVec -txn_spill_tests_files_of(HostIoIo *io, HostSharedRecord state) -{ - HostStorage storage; - TxnStatePersistence persistence; - txn_TupleVecFileU64_t listed; - HostError error; + if (file->from == 0 || file->from != frame->through) + continue; + for (size_t j = 0; j < frame->residuals.len && !found; j++) + { + TxnSpillFilesResidual *at = &STD_VEC_AT(&frame->residuals, TxnSpillFilesResidual, j); - if (!txn_state_described(state, &storage, &persistence)) - STD_OPTION_EXPECT_FAILED(1854, 52, "the state is filled"); - if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &listed, &error)) - STD_RESULT_EXPECT_FAILED(1855, 37, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - return listed.f0; + found = at->xid == file->xid && at->kind == file->kind; + if (found && (file->from > at->from || (file->from == at->from && file->number > at->number))) + *at = residual; + } + if (found) + continue; + if (!std_vec_try_reserve(&frame->residuals, 1, sizeof(residual), _Alignof(TxnSpillFilesResidual))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(&frame->residuals, &residual, sizeof(residual), _Alignof(TxnSpillFilesResidual)); + } + return true; } /* - * txn_spill_tests_written writes a test's spill file of rows of a storage's - * state by a transaction, of a number, holding its keys with no versions, and - * its filter, and returns it with its path. + * txn_spill_files_passed reports whether a listed file is left out of the + * read: of a residual's transaction and kind every file but that residual, + * any other residual, and a map's spill the newest base passed, a committed + * transaction's whose commit ends at or below the base's point or one + * marked with a passing point there. */ -static TxnSpillFile -txn_spill_tests_written(HostIoIo *io, HostSharedRecord state, uint64 xid, uint32 number, const StdBytes *keys, - size_t keys_len, StdPathBuf *path) +static bool +txn_spill_files_passed(txn_spill_files_frame_0_t *frame, const TxnSpillFile *file, bool *result) { - TxnRunsTail tail = {.tag = TXN_RUNS_TAIL_NONE}; - HostStorage storage; - TxnStatePersistence persistence; - StdPathBuf dir; - StdStr made[2]; - ChunksErrorError failed; - TxnRunsNames names; - TxnRunsWriting *writing; - TxnSpillFilter filter; - HostError error; - RepoRunFile run; - StdString name; - TxnSpillFile file; - bool some; + uint64 point = 0; + bool cached = false; - if (!txn_state_described(state, &storage, &persistence)) - STD_OPTION_EXPECT_FAILED(1866, 52, "the state is filled"); - txn_runs_spill_directory(storage.database, txn_state_own(state), &dir); - if (!std_path_parent(dir.bytes.ptr, dir.bytes.len, &made[0])) - STD_OPTION_UNWRAP_FAILED(1868, 35); - made[1] = (StdStr) + *result = false; + for (size_t j = 0; j < frame->residuals.len; j++) { - dir.bytes.ptr, dir.bytes.len - }; - for (size_t k = 0; k < lengthof(made); k++) - if (!chunks_file_create_dir(HOST_IO_IO_IO_DOORS(io), made[k].ptr, made[k].len, &failed)) - STD_RESULT_EXPECT_FAILED(1869, 48, "called `Result::unwrap()` on an `Err` value", &failed, - chunks_error_Error_Debug_fmt); - if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, state, &storage, TXN_RUNS_KIND_SPILLED_ROWS, xid, &names, &error)) - STD_RESULT_EXPECT_FAILED(1871, 68, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - writing = txn_runs_Writing_spill(dir.bytes.ptr, dir.bytes.len, &names, tail, number); - for (size_t k = 0; k < keys_len; k++) - if (!txn_runs_Writing_key(writing, io, keys[k].ptr, keys[k].len, NULL, 0, &error)) - STD_RESULT_EXPECT_FAILED(1874, 39, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_spill_Filter_of(keys_len, keys, keys_len, &filter, &error)) - STD_RESULT_EXPECT_FAILED(1876, 67, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_runs_Writing_end(writing, io, filter.words.ptr, filter.words.len, &some, &run, &error)) - STD_RESULT_EXPECT_FAILED(1877, 50, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!some) - STD_OPTION_UNWRAP_FAILED(1877, 59); - txn_runs_Writing_drop(writing); - file = (TxnSpillFile) + const TxnSpillFilesResidual *residual = &STD_VEC_AT(&frame->residuals, TxnSpillFilesResidual, j); + + if (residual->xid == file->xid && residual->kind == file->kind) + { + *result = file->from != residual->from || file->number != residual->number; + return true; + } + } + if (file->from != 0) { - .xid = xid, .kind = TXN_RUNS_KIND_SPILLED_ROWS, .number = number, .len = run.len, - .hash = run.hash, .bits = txn_spill_Filter_bits(&filter) - }; - txn_runs_spill_name(&names, number, &name); - std_path_join(dir.bytes.ptr, dir.bytes.len, name.ptr, name.len, path); - std_String_drop(&name); - std_vec_free_Checked(&filter.words, sizeof(uint64), _Alignof(uint64)); - std_vec_free_Checked(&dir.bytes, 1, 1); - return file; -} - -/* txn_spill_tests_of_742 is the list, hit, and removal case's first removal's pick: transaction 742's files. */ -static bool -txn_spill_tests_of_742(void *captures, const TxnSpillFile *file) -{ - return file->xid == 742; -} - -/* txn_spill_tests_every is the list, hit, and removal case's second removal's pick: every file. */ -static bool -txn_spill_tests_every(void *captures, const TxnSpillFile *file) -{ - return true; -} - -/* - * Bugs row 143's manifest: listed spills list newest first with what names - * their files, each file's filter, read from its own pages, hits its own keys - * and misses most others, and a removal takes exactly the files it picks out - * of the manifest, the delta's used bytes unmoved by any of it. - */ -void -test_txn_spill_tests_a_storages_spills_list_hit_and_remove(void) -{ - static const StdFmtPiece pieces[] = {STD_FMT_ARG(0), STD_FMT_LIT(" misses")}; - static const StdBytes first[] = {{(const uint8 *) "a", 1}, {(const uint8 *) "b", 1}}; - static const StdBytes second[] = {{(const uint8 *) "c", 1}}; - static const StdBytes third[] = {{(const uint8 *) "d", 1}}; - static const StdBytes *const keys[] = {first, second, third}; - static const size_t keys_lens[] = {lengthof(first), lengthof(second), lengthof(third)}; - static const uint64 xids[] = {742, 742, 800}; - static const uint32 numbers[] = {0, 1, 0}; - txn_TupleU64KindU32OptionU32_t no_replacing = {0}; - SimcTestIo test_io = simc_io(); - HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9741); - HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9741); - uint64 used = host_delta_used(); - uint64 now; - TxnSpillFile spills[3]; - StdPathBuf paths[3]; - TxnSpillFile wanted[3]; - StdVec files; - StdVec gone; - StdPathBuf manifest; - HostError error; - HostFile opened; - bool there; - bool hit; + *result = true; + return true; + } + if (frame->through == 0) + return true; + for (size_t j = 0; j < frame->passing.len && !cached; j++) + { + const std_TupleU64U64_t *passing = &STD_VEC_AT(&frame->passing, std_TupleU64U64_t, j); - for (size_t k = 0; k < lengthof(spills); k++) - spills[k] = txn_spill_tests_written(io, state, xids[k], numbers[k], keys[k], keys_lens[k], &paths[k]); - for (size_t k = 0; k < lengthof(spills); k++) - if (!txn_spill_linked(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &spills[k], 1, false, no_replacing, - &error)) - STD_RESULT_EXPECT_FAILED(1899, 63, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - now = host_delta_used(); - STD_ASSERT_EQ_MSG(1901, 9, now == used, &now, std_u64_Debug_fmt, &used, std_u64_Debug_fmt, - "the manifest takes none of the delta"); - files = txn_spill_tests_files_of(io, state); - wanted[0] = spills[2]; - wanted[1] = spills[1]; - wanted[2] = spills[0]; - txn_spill_tests_files_are(1902, 9, &files, wanted, 3); - for (size_t k = 0; k < lengthof(spills); k++) + cached = passing->f0 == file->xid; + if (cached) + point = passing->f1; + } + if (!cached) { - size_t misses = 0; + std_TupleU64U64_t passing = {file->xid, file->passed}; - hit = true; - for (size_t j = 0; hit && j < keys_lens[k]; j++) - if (!txn_spill_hits(io, &paths[k], &spills[k], keys[k][j].ptr, keys[k][j].len, &hit, &error)) - STD_RESULT_EXPECT_FAILED(1904, 71, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(1904, 13, hit, "keys.iter().all(|key| hits(io, (path, file), key).unwrap())"); - for (uint32 other = 0; other < 100; other++) + if (file->passed == 0) { - uint8 key[4]; + HostXid xid = {file->xid}; + TxnStatusCommit commit; - std_u32_to_be_bytes(other, key); - if (!txn_spill_hits(io, &paths[k], &spills[k], key, sizeof(key), &hit, &error)) - STD_RESULT_EXPECT_FAILED(1905, 99, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!hit) - misses++; + if (!txn_status_committed(frame->io, xid, &commit, frame->error)) + return false; + if (commit.tag == TXN_STATUS_COMMIT_AT) + passing.f1 = commit.u.at; } - if (misses < 90) + if (!std_vec_try_reserve(&frame->passing, 1, sizeof(passing), _Alignof(std_TupleU64U64_t))) { - StdFmtArg args[1] = {{&misses, std_usize_Display_fmt}}; - StdFmtArguments message = {pieces, lengthof(pieces), args}; - - std_panic_fmt(RUST_FILE, 1906, 13, &message); + *frame->error = host_Error_out_of_memory(); + return false; } + std_vec_push_Checked(&frame->passing, &passing, sizeof(passing), _Alignof(std_TupleU64U64_t)); + point = passing.f1; } - if (!txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, txn_spill_tests_of_742, NULL, &gone, - &error)) - STD_RESULT_EXPECT_FAILED(1908, 79, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - wanted[0] = spills[0]; - wanted[1] = spills[1]; - txn_spill_tests_files_are(1909, 9, &gone, wanted, 2); - files = txn_spill_tests_files_of(io, state); - txn_spill_tests_files_are(1910, 9, &files, &spills[2], 1); - if (!txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, txn_spill_tests_every, NULL, &gone, - &error)) - STD_RESULT_EXPECT_FAILED(1911, 54, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - std_vec_free_Checked(&gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - files = txn_spill_tests_files_of(io, state); - STD_ASSERT(1912, 9, files.len == 0, "files_of(io, state).is_empty()"); - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - txn_spill_manifest(&TXN_SPILL_STATE_HOME_OPS, state, &storage, &manifest); - if (!host_open_if_there(HOST_IO_IO_READ_DOORS(io), manifest.bytes.ptr, manifest.bytes.len, &there, &opened, - &error)) - STD_RESULT_EXPECT_FAILED(1913, 75, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (there) - host_File_drop(&opened); - std_vec_free_Checked(&manifest.bytes, 1, 1); - STD_ASSERT(1913, 9, !there, "host::open_if_there(io, &manifest(state, &storage(9741))).unwrap().is_none()"); - for (size_t k = 0; k < lengthof(paths); k++) - std_vec_free_Checked(&paths[k].bytes, 1, 1); - host_shared_Held_drop(&held); - simc_TestIo_drop(&test_io); + *result = point != 0 && point <= frame->through; + return true; } -/* txn_spill_tests_pended_are asserts at its line that a storage's pended spills are only a wanted one, or are none with an assert!'s message. */ +/* txn_spill_files_body_0 is files' body. */ static void -txn_spill_tests_pended_are(uint32 line, uint32 unwrap_col, HostSharedRecord state, const HostStorage *storage, - const txn_TupleU64KindU32OptionU32_t *wanted, const char *empty) +txn_spill_files_body_0(void *frame_) { - StdVec wanted_vec = {(void *) wanted, wanted != NULL, wanted != NULL}; - StdVec pended; - HostError error; - bool eq; + txn_spill_files_frame_0_t *frame = frame_; - if (!txn_spill_pended(&TXN_SPILL_STATE_HOME_OPS, state, storage, &pended, &error)) - STD_RESULT_EXPECT_FAILED(line, unwrap_col, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - eq = pended.len == wanted_vec.len; - if (eq && wanted != NULL) + if (!txn_spill_consistent(frame->home, frame->io, frame->state, frame->storage, &frame->slotted, + &frame->slotted_live, &frame->listed, &frame->listed_live, frame->error)) { - const txn_TupleU64KindU32OptionU32_t *slotted = pended.ptr; + txn_spill_files_pad_0(frame); + return; + } + txn_runs_spill_directory(frame->storage->database, frame->home->own(frame->state), &frame->dir); + frame->dir_live = true; + std_vec_new(frame->result, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); + frame->result_live = true; + std_vec_new(&frame->passing, sizeof(std_TupleU64U64_t), _Alignof(std_TupleU64U64_t)); + frame->passing_live = true; + if (((frame->kind == TXN_RUNS_KIND_SPILLED_FREE || frame->kind == TXN_RUNS_KIND_SPILLED_VALUES) && + !frame->home->through(frame->io, frame->state, &frame->through, frame->error)) || + !txn_spill_files_residuals(frame)) + { + txn_spill_files_pad_0(frame); + return; + } + for (size_t k = 0; k < frame->slotted.len; k++) + { + const txn_TupleU64KindU32OptionU32_t *slot = &STD_VEC_AT(&frame->slotted, txn_TupleU64KindU32OptionU32_t, k); + TxnSpillReading reading = {slot->f0, slot->f1, slot->f2}; - eq = slotted->f0 == wanted->f0 && slotted->f1 == wanted->f1 && slotted->f2 == wanted->f2 && - slotted->has_f3 == wanted->has_f3 && (!slotted->has_f3 || slotted->f3 == wanted->f3); + if (txn_spill_tree(slot->f1, frame->kind) && !txn_spill_files_push(frame, &reading)) + { + txn_spill_files_pad_0(frame); + return; + } } - if (!eq && wanted == NULL) - std_panic(RUST_FILE, line, 9, empty, strlen(empty)); - STD_ASSERT_EQ(line, 9, eq, &pended, txn_spill_tests_slotteds_Debug_fmt, &wanted_vec, - txn_spill_tests_slotteds_Debug_fmt); - std_vec_free_Checked(&pended, sizeof(txn_TupleU64KindU32OptionU32_t), _Alignof(txn_TupleU64KindU32OptionU32_t)); -} + for (size_t k = 0; k < frame->listed.f0.len; k++) + { + const TxnSpillFile *file = &STD_VEC_AT(&frame->listed.f0, TxnSpillFile, k); + TxnSpillReading reading = {file->xid, file->kind, file->number, true, file->len}; + bool seen = false; + bool hit = true; + bool passed; -/* txn_spill_tests_pending_is asserts at its place that the process's slot names a pended spill, or none. */ -static void -txn_spill_tests_pending_is(uint32 line, HostIoIo *io, const TxnSpillPended *wanted) -{ - TxnSpillPended pended; - StdOptionRef left; - StdOptionRef right = {wanted != NULL, STD_ITEM(wanted, txn_spill_tests_Pended_Debug_fmt)}; - HostError error; - bool some; - bool eq; + for (size_t j = 0; j < frame->result->len && !seen; j++) + { + const TxnSpillReading *found = &STD_VEC_AT(frame->result, TxnSpillReading, j); - if (!txn_spill_pending(io, &some, &pended, &error)) - STD_RESULT_EXPECT_FAILED(line, 32, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - left = (StdOptionRef) - { - some, STD_ITEM(&pended, txn_spill_tests_Pended_Debug_fmt) - }; - eq = some == (wanted != NULL); - if (eq && some) - eq = host_Storage_PartialEq_eq(&pended.storage, &wanted->storage) && pended.xid == wanted->xid && - pended.kind == wanted->kind && pended.number == wanted->number && pended.tupled == wanted->tupled && - pended.has_sources == wanted->has_sources && (!pended.has_sources || pended.sources == wanted->sources); - STD_ASSERT_EQ(line, 9, eq, &left, std_option_Debug_fmt, &right, std_option_Debug_fmt); + seen = found->xid == file->xid && found->number == file->number; + } + if (!txn_spill_tree(file->kind, frame->kind) || seen || + txn_spill_replaced_by(frame->slotted.ptr, frame->slotted.len, file)) + continue; + if (!txn_spill_files_passed(frame, file, &passed)) + { + txn_spill_files_pad_0(frame); + return; + } + if (passed) + continue; + if ((frame->has_key && !txn_spill_files_hit(frame, file, &hit)) || + (hit && !txn_spill_files_push(frame, &reading))) + { + txn_spill_files_pad_0(frame); + return; + } + } + frame->result_live = false; + frame->ok = true; + txn_spill_files_pad_0(frame); } /* - * Bugs row 74's pending slot: a process's pending spill is found by a reader - * of its storage, not of another, while it is pending, and a second pend - * before it settles is refused. + * txn_spill_files returns a storage's spill files of a kind's tree: those + * processes' slots name first, then the listed newest-listed first, each + * once, and, for a reader that looks for a key, only the listed whose + * filters it hits. A map's listed spill the newest base has passed (the + * home's through) is left out: a committed transaction's whose commit ends + * at or below the base's fold point, or an aborted one's marked with an end + * there, which the fold that installed the base applied or left in a + * residual, so a reader sees its versions once. A residual is read only + * while that point is the residual's own, and of a transaction's residuals + * of a kind with that point only the one with the greatest number, which + * supersedes the rest and every other file of its transaction and kind + * (bugs row 228); a residual whose fold's install failed is never read, + * since no base has its point. */ -void -test_txn_spill_tests_a_pending_spill_is_found_by_its_storages_readers_until_it_settles(void) +static bool +txn_spill_files(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, const HostStorage *storage, + TxnRunsKind kind, bool has_key, const uint8 *key, size_t key_len, StdVec *result, HostError *error) { - SimcTestIo test_io = simc_io(); - HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9742); - HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9742); - HostStorage other = storage; - txn_TupleU64KindU32OptionU32_t slotted = {.f0 = 742, .f1 = TXN_RUNS_KIND_SPILLED_ROWS, .f2 = 3}; - TxnSpillPended pended = {.storage = storage, .xid = 742, .kind = TXN_RUNS_KIND_SPILLED_ROWS, .number = 3}; - TxnSpillFile file; - HostError error; - bool refused; + txn_spill_files_frame_0_t frame = {.home = home, .io = io, .state = state, .storage = storage, .kind = kind, + .has_key = has_key, .key = key, .key_len = key_len, .result = result, .error = error}; - other.relation = 16385; - txn_spill_tests_pended_are(1925, 41, state, &storage, NULL, "assertion failed: pended(state, &storage).unwrap().is_empty()"); - file = txn_spill_tests_file(742, 3); - if (!txn_spill_pend(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &file, false, false, 0, &error)) - STD_RESULT_EXPECT_FAILED(1926, 65, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_spill_tests_pended_are(1927, 44, state, &storage, &slotted, NULL); - txn_spill_tests_pended_are(1928, 39, state, &other, NULL, - "assertion failed: pended(state, &other).unwrap().is_empty()"); - txn_spill_tests_pending_is(1929, io, &pended); - file = txn_spill_tests_file(742, 4); - refused = !txn_spill_pend(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &file, false, false, 0, &error); - if (refused) - host_Error_drop(&error); - STD_ASSERT(1930, 9, refused, "pend(io, state, &storage, (&file(742, 4), false), None).is_err()"); - if (!txn_spill_settled(&TXN_SPILL_STATE_HOME_OPS, io, &state, &error)) - STD_RESULT_EXPECT_FAILED(1931, 34, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_spill_tests_pended_are(1932, 41, state, &storage, NULL, "assertion failed: pended(state, &storage).unwrap().is_empty()"); - txn_spill_tests_pending_is(1933, io, NULL); - host_shared_Held_drop(&held); - simc_TestIo_drop(&test_io); + std_landing(&frame, txn_spill_files_body_0, txn_spill_files_pad_0); + return frame.ok; } -/* txn_spill_tests_numbered returns the four-byte big-endian keys of a count of numbers from zero, and their slices. */ +/* txn_spill_looked_up_captures_t is a lookup's captures: the key and the records it answers. */ +typedef struct txn_spill_looked_up_captures_t +{ + const uint8 *key; + size_t key_len; + StdVec *records; + bool *records_live; +} txn_spill_looked_up_captures_t; + +/* txn_spill_looked_up_closure_0 is a lookup's read: the key's records in the file. */ +static bool +txn_spill_looked_up_closure_0(void *captures, HostIoIo *io, TxnSpillRead *read, HostError *error) +{ + txn_spill_looked_up_captures_t *c = captures; + + if (!txn_spill_Read_looked_up(read, io, c->key, c->key_len, c->records, error)) + return false; + *c->records_live = true; + return true; +} + +/* txn_spill_versions_frame_0_t is versions' state across its pad: the readings, the spills' directory, a file's path and records, and the versions found, while live. */ +typedef struct txn_spill_versions_frame_0_t +{ + const TxnSpillHomeOps *home; + HostIoIo *io; + HostSharedRecord state; + TxnRunsKind kind; + const uint8 *key; + size_t key_len; + + /* + * listed, when given, takes the home's take generation once the spills + * are listed. + */ + uint64 *listed; + StdVec *result; + HostError *error; + bool ok; + bool result_live; + HostStorage storage; + StdVec files; + bool files_live; + StdPathBuf dir; + bool dir_live; + StdString name; + bool name_live; + StdPathBuf path; + bool path_live; + /* records is a Vec>, of which moved are taken. */ + StdVec records; + bool records_live; + size_t moved; +} txn_spill_versions_frame_0_t; + +/* txn_spill_versions_pad_0 is versions' pad: what it holds, while live. */ static void -txn_spill_tests_numbered(uint32 count, StdVec *bytes, StdVec *keys) +txn_spill_versions_pad_0(void *frame_) { - std_vec_from_elem_zeroed(bytes, count, 4, 1); - std_vec_from_elem_zeroed(keys, count, sizeof(StdBytes), _Alignof(StdBytes)); - for (uint32 key = 0; key < count; key++) - { - uint8 *at = (uint8 *) bytes->ptr + 4 * (size_t) key; + txn_spill_versions_frame_0_t *frame = frame_; - std_u32_to_be_bytes(key, at); - STD_VEC_AT(keys, StdBytes, key) = (StdBytes) - { - at, 4 - }; + if (frame->records_live) + { + frame->records_live = false; + for (size_t k = frame->moved; k < frame->records.len; k++) + std_vec_free_Checked(&STD_VEC_AT(&frame->records, StdVec, k), 1, 1); + std_vec_free_Checked(&frame->records, sizeof(StdVec), _Alignof(StdVec)); + } + if (frame->path_live) + { + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + } + if (frame->name_live) + { + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + } + if (frame->dir_live) + { + frame->dir_live = false; + std_vec_free_Checked(&frame->dir.bytes, 1, 1); + } + if (frame->files_live) + { + frame->files_live = false; + std_vec_free_Checked(&frame->files, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); + } + if (frame->result_live) + { + frame->result_live = false; + txn_memtable_version_versions_drop(frame->result, 0); } } -/* - * Bugs row 74's filter: every key a filter was built of hits it, and of a - * thousand other keys about one in a hundred does, at most three in a - * hundred. - */ -void -test_txn_spill_tests_a_filter_hits_its_keys_and_few_others(void) +/* txn_spill_versions_file reads one file's versions of the key into the frame's result, each by its handle. */ +static bool +txn_spill_versions_file(txn_spill_versions_frame_0_t *frame, const TxnSpillReading *reading) { - static const StdFmtPiece pieces[] = {STD_FMT_ARG(0), STD_FMT_LIT(" false hits")}; - StdVec bytes; - StdVec keys; - TxnSpillFilter filter; - HostError error; - uint64 bits; - uint64 wanted = 10048; - size_t false_hits = 0; - bool all = true; + TxnRunsNames names; + txn_spill_looked_up_captures_t captures = {frame->key, frame->key_len, &frame->records, &frame->records_live}; - txn_spill_tests_numbered(1000, &bytes, &keys); - if (!txn_spill_Filter_of(keys.len, keys.ptr, keys.len, &filter, &error)) - STD_RESULT_EXPECT_FAILED(1941, 78, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - bits = txn_spill_Filter_bits(&filter); - STD_ASSERT_EQ(1942, 9, bits == wanted, &bits, std_u64_Debug_fmt, &wanted, std_u64_Debug_fmt); - for (size_t k = 0; all && k < keys.len; k++) - all = txn_spill_Filter_hits(&filter, STD_VEC_AT(&keys, StdBytes, k).ptr, 4); - STD_ASSERT(1943, 9, all, "keys.iter().all(|key| filter.hits(key))"); - for (uint32 key = 1000; key < 2000; key++) + if (!txn_spill_names(frame->home, frame->state, &frame->storage, reading->kind, reading->xid, &names, + frame->error)) + return false; + txn_runs_spill_name(&names, reading->number, &frame->name); + frame->name_live = true; + std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, + &frame->path); + frame->path_live = true; + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + frame->moved = 0; + if (!txn_spill_reading(frame->io, &frame->path, &names, reading->has_len, reading->len, + txn_spill_looked_up_closure_0, &captures, frame->error)) + return false; + if (!std_vec_try_reserve(frame->result, frame->records.len, sizeof(TxnMemtableVersionVersion), + _Alignof(TxnMemtableVersionVersion))) { - uint8 be[4]; - - std_u32_to_be_bytes(key, be); - if (txn_spill_Filter_hits(&filter, be, sizeof(be))) - false_hits++; + *frame->error = host_Error_out_of_memory(); + return false; } - if (false_hits > 30) + while (frame->moved < frame->records.len) { - StdFmtArg args[1] = {{&false_hits, std_usize_Display_fmt}}; - StdFmtArguments message = {pieces, lengthof(pieces), args}; + uint32 place = (uint32) frame->moved; + TxnMemtableVersionVersion version; - std_panic_fmt(RUST_FILE, 1945, 9, &message); + version = txn_memtable_version_Version_spilled(txn_memtable_version_Handle_spilled(reading->xid, + reading->number, place), + &STD_VEC_AT(&frame->records, StdVec, frame->moved)); + frame->moved++; + std_vec_push_Checked(frame->result, &version, sizeof(version), _Alignof(TxnMemtableVersionVersion)); } - std_vec_free_Checked(&filter.words, sizeof(uint64), _Alignof(uint64)); - std_vec_free_Checked(&keys, sizeof(StdBytes), _Alignof(StdBytes)); - std_vec_free_Checked(&bytes, 4, 1); + frame->records_live = false; + std_vec_free_Checked(&frame->records, sizeof(StdVec), _Alignof(StdVec)); + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + return true; } -/* - * Bugs row 143's filter pages: a file's filter past a page's bits is written - * in several pages, each of the most words but the last, which the file's - * trailer places, and a key whose bits fall in each page hits. - */ -void -test_txn_spill_tests_a_large_filter_takes_several_pages(void) +/* txn_spill_versions_body_0 is versions' body. */ +static void +txn_spill_versions_body_0(void *frame_) { - SimcTestIo test_io = simc_io(); - HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9517); - HostSharedRecord state = host_shared_Held_record(&held); - StdVec bytes; - StdVec listed; - StdPathBuf path; - TxnSpillFile file; - HostFile opened; - ChunksErrorError failed; - HostError error; - StdVec page; - size_t pages[3]; - size_t wanted[3]; - StdVec pages_vec = {pages, 3, 3}; - StdVec wanted_vec = {wanted, 3, 3}; - uint64 count; - uint64 three = 3; - bool refused; - bool hit = true; + txn_spill_versions_frame_0_t *frame = frame_; + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + TxnStatePersistence persistence; - txn_spill_tests_numbered(14000, &bytes, &listed); - file = txn_spill_tests_written(io, state, 744, 0, listed.ptr, listed.len, &path); - count = std_div_ceil_u64(file.bits, TXN_SPILL_PAGE_BITS, RUST_FILE, 1959, 30); - STD_ASSERT_EQ(1959, 9, count == three, &count, std_u64_Debug_fmt, &three, std_u64_Debug_fmt); - if (!chunks_file_open_rw(HOST_IO_IO_IO_DOORS(io), path.bytes.ptr, path.bytes.len, false, &opened, &failed)) - STD_RESULT_EXPECT_FAILED(1960, 62, "called `Result::unwrap()` on an `Err` value", &failed, - chunks_error_Error_Debug_fmt); - for (uint64 at = 0; at < 3; at++) + std_vec_new(frame->result, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + if ((host_shared_Record_atomic(frame->state, frame->home->pended, load) == 0 && + host_shared_Record_atomic(frame->state, frame->home->listed, load) == 0) || + !frame->home->described(frame->state, &frame->storage, &persistence)) { - if (!txn_runs_spill_filter(io, &opened, file.len, file.bits, at, &page, &error)) - STD_RESULT_EXPECT_FAILED(1962, 92, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - pages[at] = page.len; - std_vec_free_Checked(&page, 1, 1); + frame->ok = true; + return; } - wanted[0] = TXN_RUNS_FILTER_WORDS * 8; - wanted[1] = TXN_RUNS_FILTER_WORDS * 8; - wanted[2] = (size_t) (file.bits - 2 * TXN_SPILL_PAGE_BITS) / 8; - STD_ASSERT_EQ(1963, 9, pages[0] == wanted[0] && pages[1] == wanted[1] && pages[2] == wanted[2], &pages_vec, - txn_memtable_table_tests_usizes_Debug_fmt, &wanted_vec, txn_memtable_table_tests_usizes_Debug_fmt); - refused = !txn_runs_spill_filter(io, &opened, file.len, file.bits, 3, &page, &error); - if (refused) - host_Error_drop(&error); - else - std_vec_free_Checked(&page, 1, 1); - STD_ASSERT(1964, 9, refused, "runs::spill_filter(io, &opened, (file.len, file.bits), 3).is_err()"); - for (size_t k = 0; hit && k < listed.len; k++) - if (!txn_spill_hits(io, &path, &file, STD_VEC_AT(&listed, StdBytes, k).ptr, 4, &hit, &error)) - STD_RESULT_EXPECT_FAILED(1965, 71, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(1965, 9, hit, "listed.iter().all(|key| hits(io, (&path, &file), key).unwrap())"); - host_File_drop(&opened); - std_vec_free_Checked(&path.bytes, 1, 1); - std_vec_free_Checked(&listed, sizeof(StdBytes), _Alignof(StdBytes)); - std_vec_free_Checked(&bytes, 4, 1); - host_shared_Held_drop(&held); - simc_TestIo_drop(&test_io); + frame->result_live = true; + txn_runs_spill_directory(frame->storage.database, frame->home->own(frame->state), &frame->dir); + frame->dir_live = true; + if (!txn_spill_files(frame->home, frame->io, frame->state, &frame->storage, frame->kind, true, frame->key, + frame->key_len, &frame->files, frame->error)) + { + txn_spill_versions_pad_0(frame); + return; + } + frame->files_live = true; + if (frame->listed != NULL) + *frame->listed = host_shared_Record_atomic(frame->state, frame->home->taken, load); + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-spill-listed", frame->error)) + { + txn_spill_versions_pad_0(frame); + return; + } + for (size_t k = 0; k < frame->files.len; k++) + if (!txn_spill_versions_file(frame, &STD_VEC_AT(&frame->files, TxnSpillReading, k))) + { + txn_spill_versions_pad_0(frame); + return; + } + frame->result_live = false; + frame->ok = true; + txn_spill_versions_pad_0(frame); } -/* txn_spill_tests_read_t is a spill case's read key: its key and its versions' values, newest first. */ -typedef struct txn_spill_tests_read_t -{ - StdBytes key; - /* values is a Vec>. */ - StdVec values; -} txn_spill_tests_read_t; - -/* txn_spill_tests_read_Debug_fmt is a read key's Debug, the tuple (key, values). */ -static bool -txn_spill_tests_read_Debug_fmt(const void *value, StdFormatter *f) +/* + * txn_spill_versions returns a key's versions in a storage's spills of a + * kind, newest first: those of the file a process's slot names, then those + * of the listed files whose filters the key hits, newest-listed first, each + * file once, read under a hold, storing, given where, the home's take + * generation once it listed them. + */ +bool +txn_spill_versions(const TxnSpillHomeOps *home, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, + TxnRunsKind kind, const uint8 *key, size_t key_len, uint64 *listed, StdVec *result, + HostError *error) { - const txn_spill_tests_read_t *read = value; - StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&read->key, std_bytes_Debug_fmt), - STD_ITEM(&read->values, txn_memtable_table_tests_keys_Debug_fmt)); + txn_spill_versions_frame_0_t frame = {.home = home, .io = io, .state = state, .kind = kind, .key = key, + .key_len = key_len, .listed = listed, .result = result, .error = error}; - return std_tuple_Debug_fmt(&tuple, f); + std_landing(&frame, txn_spill_versions_body_0, txn_spill_versions_pad_0); + return frame.ok; } -/* txn_spill_tests_reads_Debug_fmt is a Vec of read keys' Debug. */ -static bool -txn_spill_tests_reads_Debug_fmt(const void *value, StdFormatter *f) +/* txn_spill_next_captures_t is next's read's captures: the place, the way, and the key it answers. */ +typedef struct txn_spill_next_captures_t { - const StdVec *reads = value; - StdDebugList list = std_fmt_debug_list(f); - - for (size_t k = 0; k < reads->len; k++) - std_DebugList_entry(&list, &STD_VEC_AT(reads, txn_spill_tests_read_t, k), txn_spill_tests_read_Debug_fmt); - return std_DebugList_finish(&list); -} + bool has_after; + const uint8 *after; + size_t after_len; + TxnMemtableTableDirection way; + bool *some; + txn_TupleVecU8VecVecU8_t *result; + bool *result_live; +} txn_spill_next_captures_t; -/* txn_spill_tests_reads_eq reports whether two Vecs of read keys are equal. */ +/* txn_spill_next_closure_0 is next's read: the file's first key past the place. */ static bool -txn_spill_tests_reads_eq(const StdVec *a, const StdVec *b) +txn_spill_next_closure_0(void *captures, HostIoIo *io, TxnSpillRead *read, HostError *error) { - if (a->len != b->len) - return false; - for (size_t k = 0; k < a->len; k++) - { - const txn_spill_tests_read_t *x = &STD_VEC_AT(a, txn_spill_tests_read_t, k); - const txn_spill_tests_read_t *y = &STD_VEC_AT(b, txn_spill_tests_read_t, k); + txn_spill_next_captures_t *c = captures; - if (x->key.len != y->key.len || memcmp(x->key.ptr, y->key.ptr, x->key.len) != 0 || - !txn_memtable_table_tests_keys_eq(&x->values, &y->values)) - return false; - } + if (!txn_spill_Read_next(read, io, c->has_after, c->after, c->after_len, c->way, c->some, c->result, error)) + return false; + *c->result_live = *c->some; return true; } -/* txn_spill_tests_reads_free frees a Vec of read keys and their values. */ -static void -txn_spill_tests_reads_free(StdVec *reads) +/* txn_spill_next_frame_0_t is next's state across its pad: the readings, the spills' directory, a file's path and key, and the key found, while live. */ +typedef struct txn_spill_next_frame_0_t { - for (size_t k = 0; k < reads->len; k++) - txn_memtable_table_tests_keys_free(&STD_VEC_AT(reads, txn_spill_tests_read_t, k).values); - std_vec_free_Checked(reads, sizeof(txn_spill_tests_read_t), _Alignof(txn_spill_tests_read_t)); -} + const TxnSpillHomeOps *home; + HostIoIo *io; + HostSharedRecord state; + TxnRunsKind kind; + bool has_after; + const uint8 *after; + size_t after_len; + TxnMemtableTableDirection way; -/* txn_spill_tests_value returns a version's value, as a spill case's match on its owned op reads it: a put's value, or empty for any other op. */ -static StdVec -txn_spill_tests_value(const TxnMemtableVersionRecord *record) -{ - TxnMemtableVersionOpRef op = txn_memtable_version_Record_op(record); - TxnMemtableOp owned = txn_memtable_version_OpRef_owned(&op); - StdVec value; + /* + * listed, when given, takes the home's take generation once the spills + * are listed. + */ + uint64 *listed; + bool *some; + txn_TupleVecU8VecVersion_t *result; + HostError *error; + bool ok; + bool result_live; + HostStorage storage; + StdVec files; + bool files_live; + StdPathBuf dir; + bool dir_live; + StdString name; + bool name_live; + StdPathBuf path; + bool path_live; + txn_TupleVecU8VecVecU8_t spilled; + bool spilled_live; + size_t moved; +} txn_spill_next_frame_0_t; - if (owned.tag != TXN_MEMTABLE_OP_PUT) +/* txn_spill_next_spilled_drop drops a file's key next read, its records not moved, while live. */ +static void +txn_spill_next_spilled_drop(txn_spill_next_frame_0_t *frame) +{ + if (frame->spilled_live) { - txn_memtable_Op_drop(&owned); - std_vec_new(&value, 1, 1); - return value; + frame->spilled_live = false; + std_vec_free_Checked(&frame->spilled.f0, 1, 1); + for (size_t k = frame->moved; k < frame->spilled.f1.len; k++) + std_vec_free_Checked(&STD_VEC_AT(&frame->spilled.f1, StdVec, k), 1, 1); + std_vec_free_Checked(&frame->spilled.f1, sizeof(StdVec), _Alignof(StdVec)); } - value = owned.u.put.value; - std_vec_new(&owned.u.put.value, 1, 1); - txn_memtable_Op_drop(&owned); - return value; -} - -/* txn_spill_tests_versioned_t is a spill case's chain entry: a version's writer's xid and its value. */ -typedef struct txn_spill_tests_versioned_t -{ - uint64 xid; - /* value is a Vec. */ - StdVec value; -} txn_spill_tests_versioned_t; - -/* txn_spill_tests_versioned_Debug_fmt is a chain entry's Debug, the tuple (xid, value). */ -static bool -txn_spill_tests_versioned_Debug_fmt(const void *value, StdFormatter *f) -{ - const txn_spill_tests_versioned_t *versioned = value; - StdBytes bytes = {versioned->value.ptr, versioned->value.len}; - StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&versioned->xid, std_u64_Debug_fmt), - STD_ITEM(&bytes, std_bytes_Debug_fmt)); - - return std_tuple_Debug_fmt(&tuple, f); -} - -/* txn_spill_tests_chain_Debug_fmt is a Vec of chain entries' Debug. */ -static bool -txn_spill_tests_chain_Debug_fmt(const void *value, StdFormatter *f) -{ - const StdVec *chain = value; - StdDebugList list = std_fmt_debug_list(f); - - for (size_t k = 0; k < chain->len; k++) - std_DebugList_entry(&list, &STD_VEC_AT(chain, txn_spill_tests_versioned_t, k), - txn_spill_tests_versioned_Debug_fmt); - return std_DebugList_finish(&list); } -/* txn_spill_tests_chain_is asserts at its line that a key's chain is a list of xids and values, and frees the chain. */ +/* txn_spill_next_pad_0 is next's pad: what it holds, while live. */ static void -txn_spill_tests_chain_is(uint32 line, StdVec *chain, const uint64 *xids, const char *const *values, size_t len) +txn_spill_next_pad_0(void *frame_) { - txn_spill_tests_versioned_t wanted[2]; - StdVec wanted_vec = {wanted, len, len}; - bool eq = chain->len == len; + txn_spill_next_frame_0_t *frame = frame_; - for (size_t k = 0; k < len; k++) + txn_spill_next_spilled_drop(frame); + if (frame->path_live) { - wanted[k].xid = xids[k]; - std_slice_to_vec_Checked(&wanted[k].value, (const uint8 *) values[k], strlen(values[k]), 1, 1); + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); } - for (size_t k = 0; eq && k < len; k++) + if (frame->name_live) { - const txn_spill_tests_versioned_t *entry = &STD_VEC_AT(chain, txn_spill_tests_versioned_t, k); - - eq = entry->xid == wanted[k].xid && entry->value.len == wanted[k].value.len && - memcmp(entry->value.ptr, wanted[k].value.ptr, entry->value.len) == 0; + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + } + if (frame->dir_live) + { + frame->dir_live = false; + std_vec_free_Checked(&frame->dir.bytes, 1, 1); + } + if (frame->files_live) + { + frame->files_live = false; + std_vec_free_Checked(&frame->files, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); + } + if (frame->result_live) + { + frame->result_live = false; + std_vec_free_Checked(&frame->result->f0, 1, 1); + txn_memtable_version_versions_drop(&frame->result->f1, 0); } - STD_ASSERT_EQ(line, 9, eq, chain, txn_spill_tests_chain_Debug_fmt, &wanted_vec, txn_spill_tests_chain_Debug_fmt); - for (size_t k = 0; k < len; k++) - std_vec_free_Checked(&wanted[k].value, 1, 1); - for (size_t k = 0; k < chain->len; k++) - std_vec_free_Checked(&STD_VEC_AT(chain, txn_spill_tests_versioned_t, k).value, 1, 1); - std_vec_free_Checked(chain, sizeof(txn_spill_tests_versioned_t), _Alignof(txn_spill_tests_versioned_t)); } -/* txn_spill_tests_chain returns a key's versions in a state's memtable and spills, each its writer's xid and its value, as the spill case's chain closure does. */ -static StdVec -txn_spill_tests_chain(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, const char *key) +/* txn_spill_next_file reads one file's first key past the place and keeps it when it is nearer than the one found, or the same key. */ +static bool +txn_spill_next_file(txn_spill_next_frame_0_t *frame, const TxnSpillReading *reading) { - StdVec versions; - StdVec chain; - HostError error; - - if (!txn_memtable_table_versions(io, hold, state, (const uint8 *) key, strlen(key), &versions, &error)) + TxnRunsNames names; + bool some; + txn_spill_next_captures_t captures = {frame->has_after, frame->after, frame->after_len, frame->way, &some, + &frame->spilled, &frame->spilled_live}; + bool nearer = !frame->result_live; + bool same = false; + + if (!txn_spill_names(frame->home, frame->state, &frame->storage, reading->kind, reading->xid, &names, + frame->error)) + return false; + txn_runs_spill_name(&names, reading->number, &frame->name); + frame->name_live = true; + std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, + &frame->path); + frame->path_live = true; + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + frame->moved = 0; + if (!txn_spill_reading(frame->io, &frame->path, &names, reading->has_len, reading->len, txn_spill_next_closure_0, + &captures, frame->error)) + return false; + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + if (!some) + return true; + if (frame->result_live) + { + StdOrdering order = std_bytes_Ord_cmp(frame->spilled.f0.ptr, frame->spilled.f0.len, frame->result->f0.ptr, + frame->result->f0.len); + + nearer = frame->way == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD ? order == STD_ORDERING_LESS : + order == STD_ORDERING_GREATER; + same = order == STD_ORDERING_EQUAL; + } + if (nearer) + { + if (frame->result_live) + { + frame->result_live = false; + std_vec_free_Checked(&frame->result->f0, 1, 1); + txn_memtable_version_versions_drop(&frame->result->f1, 0); + } + frame->result->f0 = frame->spilled.f0; + std_vec_new(&frame->spilled.f0, 1, 1); + std_vec_new(&frame->result->f1, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + frame->result_live = true; + } + if (nearer || same) + { + if (!std_vec_try_reserve(&frame->result->f1, frame->spilled.f1.len, sizeof(TxnMemtableVersionVersion), + _Alignof(TxnMemtableVersionVersion))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + while (frame->moved < frame->spilled.f1.len) + { + uint32 place = (uint32) frame->moved; + TxnMemtableVersionVersion version; + + version = txn_memtable_version_Version_spilled(txn_memtable_version_Handle_spilled(reading->xid, + reading->number, + place), + &STD_VEC_AT(&frame->spilled.f1, StdVec, frame->moved)); + frame->moved++; + std_vec_push_Checked(&frame->result->f1, &version, sizeof(version), _Alignof(TxnMemtableVersionVersion)); + } + } + txn_spill_next_spilled_drop(frame); + return true; +} + +/* txn_spill_next_body_0 is next's body. */ +static void +txn_spill_next_body_0(void *frame_) +{ + txn_spill_next_frame_0_t *frame = frame_; + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + TxnStatePersistence persistence; + + *frame->some = false; + if ((host_shared_Record_atomic(frame->state, frame->home->pended, load) == 0 && + host_shared_Record_atomic(frame->state, frame->home->listed, load) == 0) || + !frame->home->described(frame->state, &frame->storage, &persistence)) + { + frame->ok = true; + return; + } + txn_runs_spill_directory(frame->storage.database, frame->home->own(frame->state), &frame->dir); + frame->dir_live = true; + if (!txn_spill_files(frame->home, frame->io, frame->state, &frame->storage, frame->kind, false, NULL, 0, + &frame->files, frame->error)) + { + txn_spill_next_pad_0(frame); + return; + } + frame->files_live = true; + if (frame->listed != NULL) + *frame->listed = host_shared_Record_atomic(frame->state, frame->home->taken, load); + for (size_t k = 0; k < frame->files.len; k++) + if (!txn_spill_next_file(frame, &STD_VEC_AT(&frame->files, TxnSpillReading, k))) + { + txn_spill_next_pad_0(frame); + return; + } + *frame->some = frame->result_live; + frame->result_live = false; + frame->ok = true; + txn_spill_next_pad_0(frame); +} + +/* + * txn_spill_next returns the first key of a storage's spills of a kind past + * a place, the way a scan goes, with its versions, newest first, the newer + * files' first, or nothing past the last, read under a hold; with no place + * it starts at the first key or the last, storing, given where, the home's + * take generation once it listed the spills. + */ +bool +txn_spill_next(const TxnSpillHomeOps *home, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, + TxnRunsKind kind, bool has_after, const uint8 *after, size_t after_len, TxnMemtableTableDirection way, + uint64 *listed, bool *some, txn_TupleVecU8VecVersion_t *result, HostError *error) +{ + txn_spill_next_frame_0_t frame = {.home = home, .io = io, .state = state, .kind = kind, .has_after = has_after, + .after = after, .after_len = after_len, .way = way, .listed = listed, .some = some, .result = result, + .error = error}; + + std_landing(&frame, txn_spill_next_body_0, txn_spill_next_pad_0); + return frame.ok; +} + +/* txn_spill_held_frame_0_t is the state of a read of a key's spilled versions under a hold across its pad: the hold and the versions, while live. */ +typedef struct txn_spill_held_frame_0_t +{ + const TxnSpillHomeOps *home; + HostIoIo *io; + HostSharedRecord state; + TxnRunsKind kind; + const uint8 *key; + size_t key_len; + bool (*with) (void *captures, const StdVec *versions, HostError *error); + void *captures; + HostError *error; + bool ok; + HostDeltaHold hold; + bool hold_live; + StdVec versions; + bool versions_live; +} txn_spill_held_frame_0_t; + +/* txn_spill_held_pad_0 is the pad of a read under a hold: the versions, then the hold, while live. */ +static void +txn_spill_held_pad_0(void *frame_) +{ + txn_spill_held_frame_0_t *frame = frame_; + + if (frame->versions_live) + { + frame->versions_live = false; + txn_memtable_version_versions_drop(&frame->versions, 0); + } + if (frame->hold_live) + { + frame->hold_live = false; + host_delta_Hold_drop(&frame->hold); + } +} + +/* txn_spill_held_body_0 reads the key's spilled versions under a hold and runs the step on them. */ +static void +txn_spill_held_body_0(void *frame_) +{ + txn_spill_held_frame_0_t *frame = frame_; + + if (!host_delta_hold(frame->io, &frame->hold, frame->error)) + return; + frame->hold_live = true; + if (!txn_spill_versions(frame->home, frame->io, &frame->hold, frame->state, frame->kind, frame->key, + frame->key_len, NULL, &frame->versions, frame->error)) + { + txn_spill_held_pad_0(frame); + return; + } + frame->versions_live = true; + frame->ok = frame->with(frame->captures, &frame->versions, frame->error); + txn_spill_held_pad_0(frame); +} + +/* txn_spill_held runs a step on a key's versions in a storage's spills of a kind, read under a hold it takes. */ +static bool +txn_spill_held(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, TxnRunsKind kind, const uint8 *key, + size_t key_len, bool (*with) (void *captures, const StdVec *versions, HostError *error), + void *captures, HostError *error) +{ + txn_spill_held_frame_0_t frame = {.home = home, .io = io, .state = state, .kind = kind, .key = key, + .key_len = key_len, .with = with, .captures = captures, .error = error}; + + std_landing(&frame, txn_spill_held_body_0, txn_spill_held_pad_0); + return frame.ok; +} + +/* txn_spill_twinned_captures_t is twinned's and holds' step's captures: the record's bytes or the LSN, and the answer. */ +typedef struct txn_spill_twinned_captures_t +{ + const uint8 *bytes; + size_t bytes_len; + uint64 lsn; + bool *result; +} txn_spill_twinned_captures_t; + +/* txn_spill_twinned_closure_0 is twinned's step: whether a version's record is the bytes. */ +static bool +txn_spill_twinned_closure_0(void *captures, const StdVec *versions, HostError *error) +{ + txn_spill_twinned_captures_t *c = captures; + + *c->result = false; + for (size_t k = 0; k < versions->len && !*c->result; k++) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(versions, + TxnMemtableVersionVersion, + k)); + StdBytes bytes = txn_memtable_version_Record_bytes(&record); + + *c->result = bytes.len == c->bytes_len && memcmp(bytes.ptr, c->bytes, bytes.len) == 0; + } + return true; +} + +/* txn_spill_twinned reports whether a key's spills of a kind hold a version of a record's bytes. */ +bool +txn_spill_twinned(HostIoIo *io, HostSharedRecord state, TxnRunsKind kind, const uint8 *key, size_t key_len, + const uint8 *bytes, size_t bytes_len, bool *result, HostError *error) +{ + txn_spill_twinned_captures_t captures = {bytes, bytes_len, 0, result}; + + return txn_spill_held(&TXN_SPILL_STATE_HOME_OPS, io, state, kind, key, key_len, txn_spill_twinned_closure_0, + &captures, error); +} + +/* txn_spill_holds_closure_0 is holds' step: whether a version's record carries the LSN. */ +static bool +txn_spill_holds_closure_0(void *captures, const StdVec *versions, HostError *error) +{ + txn_spill_twinned_captures_t *c = captures; + + *c->result = false; + for (size_t k = 0; k < versions->len && !*c->result; k++) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&STD_VEC_AT(versions, + TxnMemtableVersionVersion, + k)); + uint64 lsn; + + *c->result = txn_memtable_version_Record_lsn(&record, &lsn) && lsn == c->lsn; + } + return true; +} + +/* txn_spill_holds reports whether a key's spills of a kind in a home hold a version carrying the end of a WAL record. */ +bool +txn_spill_holds(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, TxnRunsKind kind, const uint8 *key, + size_t key_len, uint64 lsn, bool *result, HostError *error) +{ + txn_spill_twinned_captures_t captures = {NULL, 0, lsn, result}; + + return txn_spill_held(home, io, state, kind, key, key_len, txn_spill_holds_closure_0, &captures, error); +} + +/* txn_spill_newest_value_captures_t is newest_value's step's captures: the entry it answers. */ +typedef struct txn_spill_newest_value_captures_t +{ + bool *some; + TxnMemtableMapEntry *result; +} txn_spill_newest_value_captures_t; + +/* txn_spill_newest_value_closure_0 is newest_value's step: the newest version's put or removal. */ +static bool +txn_spill_newest_value_closure_0(void *captures, const StdVec *versions, HostError *error) +{ + txn_spill_newest_value_captures_t *c = captures; + TxnMemtableVersionRecord record; + TxnMemtableVersionOpRef op; + + *c->some = versions->len > 0; + if (!*c->some) + return true; + record = txn_memtable_version_Version_record(&STD_VEC_AT(versions, TxnMemtableVersionVersion, 0)); + op = txn_memtable_version_Record_op(&record); + if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_PUT) + { + c->result->has_value = true; + std_slice_to_vec_Checked(&c->result->value, op.u.put.value.ptr, op.u.put.value.len, 1, 1); + return true; + } + if (op.tag == TXN_MEMTABLE_VERSION_OP_REF_DELETE) + { + c->result->has_value = false; + return true; + } + *c->some = false; + *error = txn_spill_internal("a spilled map's version neither puts nor deletes"); + return false; +} + +/* txn_spill_newest_value returns a value id's newest entry in a storage's spilled value maps: nothing for an id they hold no version of, and none for a removal. */ +bool +txn_spill_newest_value(HostIoIo *io, HostSharedRecord state, uint32 id, bool *some, TxnMemtableMapEntry *result, + HostError *error) +{ + uint8 key[4]; + txn_spill_newest_value_captures_t captures = {some, result}; + + std_u32_to_be_bytes(id, key); + return txn_spill_held(&TXN_SPILL_STATE_HOME_OPS, io, state, TXN_RUNS_KIND_SPILLED_VALUES, key, sizeof(key), + txn_spill_newest_value_closure_0, &captures, error); +} + +/* txn_spill_generation returns the generation of a storage's spills' manifest, which every change of it moves. */ +uint64 +txn_spill_generation(const TxnSpillHomeOps *home, HostSharedRecord state) +{ + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + + return host_shared_Record_atomic(state, home->changed, load); +} + +/* txn_spill_chunk_versions returns a key's versions in a working state's chunk spills, newest first, as versions does. */ +bool +txn_spill_chunk_versions(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord deltas, const uint8 *key, + size_t key_len, StdVec *result, HostError *error) +{ + return txn_spill_versions(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, io, hold, deltas, TXN_RUNS_KIND_SPILLED_CHUNKS, key, + key_len, NULL, result, error); +} + +/* txn_spill_chunk_closure_0 is chunk's step: the newest spilled version's chunk. */ +static bool +txn_spill_chunk_closure_0(void *captures, const StdVec *versions, HostError *error) +{ + txn_spill_newest_value_captures_t *c = captures; + TxnMemtableVersionRecord record; + TxnMemtableVersionOpRef op; + + *c->some = versions->len > 0; + if (!*c->some) + return true; + record = txn_memtable_version_Version_record(&STD_VEC_AT(versions, TxnMemtableVersionVersion, 0)); + op = txn_memtable_version_Record_op(&record); + if (op.tag != TXN_MEMTABLE_VERSION_OP_REF_PUT) + { + *c->some = false; + *error = txn_spill_internal("a spilled chunk's version puts no chunk"); + return false; + } + std_slice_to_vec_Checked(&c->result->value, op.u.put.value.ptr, op.u.put.value.len, 1, 1); + return true; +} + +/* + * txn_spill_chunk returns the bytes of the chunk under a key in a working + * state's delta, else in its chunk spills, the newest version's, or nothing + * for a key neither holds. + */ +bool +txn_spill_chunk(HostIoIo *io, HostSharedRecord deltas, const uint8 *key, size_t key_len, bool *some, StdVec *result, + HostError *error) +{ + TxnMemtableMapEntry entry; + txn_spill_newest_value_captures_t captures = {some, &entry}; + + if (!txn_memtable_branch_get(io, deltas, key, key_len, some, result, error)) + return false; + if (*some) + return true; + if (!txn_spill_held(&TXN_SPILL_BRANCH_DELTAS_HOME_OPS, io, deltas, TXN_RUNS_KIND_SPILLED_CHUNKS, key, key_len, + txn_spill_chunk_closure_0, &captures, error)) + return false; + if (*some) + *result = entry.value; + return true; +} + +/* txn_spill_tuple_key_frame_0_t is tuple_key's state across its pad: the readings, the spills' directory, a file's path, and the values read, while live. */ +typedef struct txn_spill_tuple_key_frame_0_t +{ + HostIoIo *io; + HostSharedRecord state; + HostRow tid; + bool *some; + StdVec *result; + HostError *error; + bool ok; + HostStorage storage; + StdVec files; + bool files_live; + StdPathBuf dir; + bool dir_live; + StdString name; + bool name_live; + StdPathBuf path; + bool path_live; + /* values is a Vec>. */ + StdVec values; + bool values_live; +} txn_spill_tuple_key_frame_0_t; + +/* txn_spill_tuple_key_pad_0 is tuple_key's pad: what it holds, while live. */ +static void +txn_spill_tuple_key_pad_0(void *frame_) +{ + txn_spill_tuple_key_frame_0_t *frame = frame_; + + if (frame->values_live) + { + frame->values_live = false; + txn_spill_versions_drop(&frame->values); + } + if (frame->path_live) + { + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + } + if (frame->name_live) + { + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + } + if (frame->dir_live) + { + frame->dir_live = false; + std_vec_free_Checked(&frame->dir.bytes, 1, 1); + } + if (frame->files_live) + { + frame->files_live = false; + std_vec_free_Checked(&frame->files, sizeof(TxnSpillReading), _Alignof(TxnSpillReading)); + } +} + +/* txn_spill_tuple_key_body_0 is tuple_key's body. */ +static void +txn_spill_tuple_key_body_0(void *frame_) +{ + txn_spill_tuple_key_frame_0_t *frame = frame_; + HostSharedOp load = {.tag = HOST_SHARED_OP_LOAD}; + TxnStatePersistence persistence; + uint8 key[8]; + + *frame->some = false; + if ((host_shared_Record_atomic(frame->state, TXN_STATE_SPILLS_PENDING_INDEX, load) == 0 && + host_shared_Record_atomic(frame->state, TXN_STATE_SPILLS_INDEX, load) == 0) || + !txn_state_described(frame->state, &frame->storage, &persistence)) + { + frame->ok = true; + return; + } + txn_runs_spill_directory(frame->storage.database, txn_state_own(frame->state), &frame->dir); + frame->dir_live = true; + std_u64_to_be_bytes(frame->tid.f0, key); + if (!txn_spill_files(&TXN_SPILL_STATE_HOME_OPS, frame->io, frame->state, &frame->storage, + TXN_RUNS_KIND_SPILLED_TUPLES, true, key, sizeof(key), &frame->files, frame->error)) + { + txn_spill_tuple_key_pad_0(frame); + return; + } + frame->files_live = true; + for (size_t k = 0; k < frame->files.len; k++) + { + const TxnSpillReading *reading = &STD_VEC_AT(&frame->files, TxnSpillReading, k); + TxnRunsNames names; + txn_spill_looked_up_captures_t captures = {key, sizeof(key), &frame->values, &frame->values_live}; + + if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, frame->state, &frame->storage, reading->kind, reading->xid, + &names, frame->error)) + { + txn_spill_tuple_key_pad_0(frame); + return; + } + txn_runs_spill_name(&names, reading->number, &frame->name); + frame->name_live = true; + std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, + &frame->path); + frame->path_live = true; + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + if (!txn_spill_reading(frame->io, &frame->path, &names, reading->has_len, reading->len, + txn_spill_looked_up_closure_0, &captures, frame->error)) + { + txn_spill_tuple_key_pad_0(frame); + return; + } + frame->values_live = true; + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + if (frame->values.len > 0) + { + *frame->result = STD_VEC_AT(&frame->values, StdVec, 0); + for (size_t v = 1; v < frame->values.len; v++) + std_vec_free_Checked(&STD_VEC_AT(&frame->values, StdVec, v), 1, 1); + frame->values_live = false; + std_vec_free_Checked(&frame->values, sizeof(StdVec), _Alignof(StdVec)); + *frame->some = true; + break; + } + frame->values_live = false; + txn_spill_versions_drop(&frame->values); + } + frame->ok = true; + txn_spill_tuple_key_pad_0(frame); +} + +/* + * txn_spill_tuple_key returns the key of a tid's row in a storage's spilled + * tuple ids, the pending file's first, or nothing for a tid they don't hold, + * read under a hold. + */ +bool +txn_spill_tuple_key(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, HostRow tid, bool *some, + StdVec *result, HostError *error) +{ + txn_spill_tuple_key_frame_0_t frame = {.io = io, .state = state, .tid = tid, .some = some, .result = result, + .error = error}; + + std_landing(&frame, txn_spill_tuple_key_body_0, txn_spill_tuple_key_pad_0); + return frame.ok; +} + +/* + * TxnSpillNaming is Naming: a spill file a checkpoint names, its xid, kind, + * and number, its length and xxh3 when its manifest lists them, the point a + * fold passes it at, or 0 for none, and its residual's point, or 0 for none. + */ +typedef struct TxnSpillNaming +{ + uint64 xid; + TxnRunsKind kind; + uint32 number; + bool has_read; + uint64 len; + uint64 hash; + uint64 passed; + uint64 from; +} TxnSpillNaming; + +/* txn_spill_named_frame_0_t is named's state across its pad: the manifest's path and its writers' lock, the slots, the listed files, the namings, the aborted and crashed xids, the spills' directory, a file's path and file, and the named run files, while live. */ +typedef struct txn_spill_named_frame_0_t +{ + const TxnSpillHomeOps *home; + HostIoIo *io; + HostSharedRecord state; + const HostStorage *storage; + HostLsn redo; + StdVec *result; + HostError *error; + bool ok; + bool result_live; + StdPathBuf manifest; + bool manifest_live; + HostSharedLongGuard writing; + bool writing_live; + HostSharedLongIo long_io; + StdVec slotted; + bool slotted_live; + /* listed is a Vec, in the manifest's order. */ + StdVec listed; + bool listed_live; + /* files is a Vec. */ + StdVec files; + bool files_live; + /* aborted is a Vec of xids. */ + StdVec aborted; + bool aborted_live; + + /* + * crashed is a Vec of the xids of free space map spills a crash + * ended. + */ + StdVec crashed; + bool crashed_live; + StdPathBuf dir; + bool dir_live; + StdString name; + bool name_live; + StdPathBuf path; + bool path_live; + HostFile file; + bool file_live; +} txn_spill_named_frame_0_t; + +/* txn_spill_named_pad_0 is named's pad: what it holds, while live. */ +static void +txn_spill_named_pad_0(void *frame_) +{ + txn_spill_named_frame_0_t *frame = frame_; + + if (frame->file_live) + { + frame->file_live = false; + host_File_drop(&frame->file); + } + if (frame->path_live) + { + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + } + if (frame->name_live) + { + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + } + if (frame->dir_live) + { + frame->dir_live = false; + std_vec_free_Checked(&frame->dir.bytes, 1, 1); + } + if (frame->crashed_live) + { + frame->crashed_live = false; + std_vec_free_Checked(&frame->crashed, sizeof(uint64), _Alignof(uint64)); + } + if (frame->aborted_live) + { + frame->aborted_live = false; + std_vec_free_Checked(&frame->aborted, sizeof(uint64), _Alignof(uint64)); + } + if (frame->files_live) + { + frame->files_live = false; + std_vec_free_Checked(&frame->files, sizeof(TxnSpillNaming), _Alignof(TxnSpillNaming)); + } + if (frame->listed_live) + { + frame->listed_live = false; + std_vec_free_Checked(&frame->listed, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + } + if (frame->slotted_live) + { + frame->slotted_live = false; + std_vec_free_Checked(&frame->slotted, sizeof(txn_TupleU64KindU32OptionU32_t), + _Alignof(txn_TupleU64KindU32OptionU32_t)); + } + if (frame->writing_live) + { + frame->writing_live = false; + host_shared_LongGuard_drop(&frame->writing); + } + if (frame->manifest_live) + { + frame->manifest_live = false; + std_vec_free_Checked(&frame->manifest.bytes, 1, 1); + } + if (frame->result_live) + { + frame->result_live = false; + std_vec_free_Checked(frame->result, sizeof(RepoRunFile), _Alignof(RepoRunFile)); + } +} + +/* txn_spill_named_push adds an element to a vector, refusing it when the vector can't grow. */ +static bool +txn_spill_named_push(txn_spill_named_frame_0_t *frame, StdVec *vector, const void *elem, size_t size, size_t align) +{ + if (!std_vec_try_reserve(vector, 1, size, align)) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_vec_push_Checked(vector, elem, size, align); + return true; +} + +/* + * txn_spill_named_listed gathers, under the manifest's writers' lock, the + * namings of the listed files in the manifest's order, oldest-linked first, + * passing over those a pending merge replaces and gathering the aborted: an + * aborted transaction's files leave the naming but its free space map's, + * and a free space map's spill of a transaction a crash ended, which no + * abort record marked, is marked with the checkpoint's redo point. + */ +static bool +txn_spill_named_listed(txn_spill_named_frame_0_t *frame) +{ + for (size_t k = 0; k < frame->listed.len; k++) + { + const TxnSpillFile *file = &STD_VEC_AT(&frame->listed, TxnSpillFile, k); + bool map_free = file->kind == TXN_RUNS_KIND_SPILLED_FREE; + TxnSpillNaming naming = {file->xid, file->kind, file->number, true, file->len, file->hash, file->passed, + file->from}; + + if (txn_spill_replaced_by(frame->slotted.ptr, frame->slotted.len, file)) + continue; + if (!map_free || file->passed == 0) + { + HostXid xid = {file->xid}; + HostStatus status; + + if (!host_status(HOST_SHARED_LONG_IO_IO_DOORS(&frame->long_io), xid, &status, frame->error)) + return false; + if (status == HOST_STATUS_ABORTED && + !txn_spill_named_push(frame, map_free ? &frame->crashed : &frame->aborted, &file->xid, sizeof(uint64), + _Alignof(uint64))) + return false; + if (status == HOST_STATUS_ABORTED && !map_free) + continue; + } + if (map_free && txn_spill_has_xid(&frame->crashed, file->xid)) + naming.passed = frame->redo; + if (!txn_spill_named_push(frame, &frame->files, &naming, sizeof(naming), _Alignof(TxnSpillNaming))) + return false; + } + return true; +} + +/* txn_spill_named_read reads a pending file's length and its trailer's xxh3 from the file. */ +static bool +txn_spill_named_read(txn_spill_named_frame_0_t *frame, const TxnSpillNaming *naming, uint64 *len, uint64 *hash) +{ + TxnRunsNames names; + ChunksErrorError failed; + + if (!txn_spill_names(frame->home, frame->state, frame->storage, naming->kind, naming->xid, &names, frame->error)) + return false; + txn_runs_spill_name(&names, naming->number, &frame->name); + frame->name_live = true; + std_path_join(frame->dir.bytes.ptr, frame->dir.bytes.len, (const char *) frame->name.ptr, frame->name.len, + &frame->path); + frame->path_live = true; + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + if (!chunks_file_open_rw(HOST_IO_IO_IO_DOORS(frame->io), frame->path.bytes.ptr, frame->path.bytes.len, false, + &frame->file, &failed)) + { + *frame->error = txn_base_storage(&failed); + return false; + } + frame->file_live = true; + if (!host_size(HOST_IO_IO_READ_DOORS(frame->io), &frame->file, len, frame->error) || + !txn_runs_spill_tail_hash(frame->io, &frame->file, *len, hash, frame->error)) + return false; + frame->file_live = false; + host_File_drop(&frame->file); + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + return true; +} + +/* txn_spill_named_body_0 is named's body. */ +static void +txn_spill_named_body_0(void *frame_) +{ + txn_spill_named_frame_0_t *frame = frame_; + + std_vec_new(&frame->files, sizeof(TxnSpillNaming), _Alignof(TxnSpillNaming)); + frame->files_live = true; + std_vec_new(&frame->aborted, sizeof(uint64), _Alignof(uint64)); + frame->aborted_live = true; + std_vec_new(&frame->crashed, sizeof(uint64), _Alignof(uint64)); + frame->crashed_live = true; + txn_spill_manifest(frame->home, frame->state, frame->storage, &frame->manifest); + frame->manifest_live = true; + if (!host_shared_Record_lock_long(frame->state, HOST_IO_IO_TAKES_SHORT(frame->io), frame->home->writing, + &frame->writing, frame->error)) + { + txn_spill_named_pad_0(frame); + return; + } + frame->writing_live = true; + frame->long_io = host_shared_LongGuard_io(&frame->writing); + if (!txn_spill_pended(frame->home, frame->state, frame->storage, &frame->slotted, frame->error)) + { + txn_spill_named_pad_0(frame); + return; + } + frame->slotted_live = true; + if (!txn_spill_read(HOST_IO_IO_DOORS_READ_DOORS(HOST_SHARED_LONG_IO_IO_DOORS(&frame->long_io)), + HOST_SHARED_LONG_IO_PAUSES(&frame->long_io), &frame->manifest, &frame->listed, frame->error)) + { + txn_spill_named_pad_0(frame); + return; + } + frame->listed_live = true; + if (!txn_spill_named_listed(frame)) + { + txn_spill_named_pad_0(frame); + return; + } + if (frame->aborted.len > 0 || frame->crashed.len > 0) + { + size_t kept = 0; + + for (size_t k = 0; k < frame->listed.len; k++) + { + TxnSpillFile file = STD_VEC_AT(&frame->listed, TxnSpillFile, k); + bool map_free = file.kind == TXN_RUNS_KIND_SPILLED_FREE; + + if (!map_free && txn_spill_has_xid(&frame->aborted, file.xid)) + continue; + if (map_free && file.passed == 0 && txn_spill_has_xid(&frame->crashed, file.xid)) + file.passed = frame->redo; + STD_VEC_AT(&frame->listed, TxnSpillFile, kept++) = file; + } + frame->listed.len = kept; + if (!txn_spill_rewrote(frame->home, &frame->long_io, frame->state, &frame->manifest, &frame->listed, + frame->error)) + { + txn_spill_named_pad_0(frame); + return; + } + } + frame->writing_live = false; + host_shared_LongGuard_drop(&frame->writing); + for (size_t k = 0; k < frame->slotted.len; k++) + { + const txn_TupleU64KindU32OptionU32_t *slot = &STD_VEC_AT(&frame->slotted, txn_TupleU64KindU32OptionU32_t, k); + TxnSpillNaming naming = {slot->f0, slot->f1, slot->f2}; + bool named = false; + + for (size_t j = 0; j < frame->files.len && !named; j++) + { + const TxnSpillNaming *found = &STD_VEC_AT(&frame->files, TxnSpillNaming, j); + + named = found->xid == slot->f0 && found->kind == slot->f1 && found->number == slot->f2; + } + if (!named && !txn_spill_named_push(frame, &frame->files, &naming, sizeof(naming), _Alignof(TxnSpillNaming))) + { + txn_spill_named_pad_0(frame); + return; + } + } + txn_runs_spill_directory(frame->storage->database, frame->home->own(frame->state), &frame->dir); + frame->dir_live = true; + std_vec_new(frame->result, sizeof(RepoRunFile), _Alignof(RepoRunFile)); + frame->result_live = true; + if (!std_vec_try_reserve(frame->result, frame->files.len, sizeof(RepoRunFile), _Alignof(RepoRunFile))) + { + *frame->error = host_Error_out_of_memory(); + txn_spill_named_pad_0(frame); + return; + } + for (size_t k = 0; k < frame->files.len; k++) + { + const TxnSpillNaming *naming = &STD_VEC_AT(&frame->files, TxnSpillNaming, k); + RepoRunFile run = {.kind = (uint8) naming->kind, .tablespace = frame->storage->tablespace, + .relation = frame->storage->relation, .len = naming->len, .hash = naming->hash, .has_spilled = true, + .spilled = {naming->xid, naming->number, naming->passed != 0, naming->passed, naming->from != 0, + naming->from}}; + + if (!naming->has_read && !txn_spill_named_read(frame, naming, &run.len, &run.hash)) + { + txn_spill_named_pad_0(frame); + return; + } + std_vec_push_Checked(frame->result, &run, sizeof(run), _Alignof(RepoRunFile)); + } + frame->result_live = false; + frame->ok = true; + txn_spill_named_pad_0(frame); +} + +/* + * txn_spill_named returns the manifest's entries of a storage's spill files + * a checkpoint names, each synced at its write: the listed oldest-linked + * first, then those processes' slots name. It reads the slots and then the + * manifest under the manifest's writers' lock, so a merge pending its file + * names that file and not the sources it replaces, and a carry that took the + * lock before it logged its versions' records lists their files before this + * naming reads. A listed spill of a transaction the commit log calls + * aborted, which only redo leaves, leaves the manifest unnamed, for the + * sweep, but a free space map's, which a fold takes; a free space map's + * spill of a transaction a crash ended, which wrote no abort record to mark + * it, is marked with the checkpoint's redo point, redo, the point a fold + * passes it at. + */ +bool +txn_spill_named(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, const HostStorage *storage, + HostLsn redo, StdVec *result, HostError *error) +{ + txn_spill_named_frame_0_t frame = {.home = home, .io = io, .state = state, .storage = storage, .redo = redo, + .result = result, .error = error}; + + std_landing(&frame, txn_spill_named_body_0, txn_spill_named_pad_0); + return frame.ok; +} + +/* txn_spill_settled_from_frame_0_t is settled_from's state across its pad: the spill's path, bytes, and decoding, its twins in the memtable, the hold and a key's chain, the tuple-id file's bytes and rows, and its tids, while live. */ +typedef struct txn_spill_settled_from_frame_0_t +{ + const TxnSpillHomeOps *home; + HostIoIo *io; + HostSharedRecord record; + const TxnSpillPended *pended; + HostError *error; + bool ok; + StdPathBuf path; + bool path_live; + StdString name; + bool name_live; + StdVec bytes; + bool bytes_live; + TxnRunsDecoded decoded; + bool decoded_live; + /* twins is a Vec<(Vec, Vec)>. */ + StdVec twins; + bool twins_live; + HostDeltaHold hold; + bool hold_live; + /* chain is a Vec. */ + StdVec chain; + bool chain_live; + /* handles is a Vec of a key's twins. */ + StdVec handles; + bool handles_live; + StdVec tuple_bytes; + bool tuple_bytes_live; + /* rows is decode_tuples' Vec<(u64, &[u8])>. */ + StdVec rows; + bool rows_live; + /* tids is a Vec. */ + StdVec tids; + bool tids_live; +} txn_spill_settled_from_frame_0_t; + +/* txn_spill_settled_from_pad_0 is settled_from's pad: what it holds, while live. */ +static void +txn_spill_settled_from_pad_0(void *frame_) +{ + txn_spill_settled_from_frame_0_t *frame = frame_; + + if (frame->tids_live) + { + frame->tids_live = false; + std_vec_free_Checked(&frame->tids, sizeof(uint64), _Alignof(uint64)); + } + if (frame->rows_live) + { + frame->rows_live = false; + std_vec_free_Checked(&frame->rows, sizeof(txn_TupleU64RefSliceU8_t), _Alignof(txn_TupleU64RefSliceU8_t)); + } + if (frame->tuple_bytes_live) + { + frame->tuple_bytes_live = false; + std_vec_free_Checked(&frame->tuple_bytes, 1, 1); + } + if (frame->handles_live) + { + frame->handles_live = false; + std_vec_free_Checked(&frame->handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + } + if (frame->chain_live) + { + frame->chain_live = false; + txn_memtable_version_versions_drop(&frame->chain, 0); + } + if (frame->hold_live) + { + frame->hold_live = false; + host_delta_Hold_drop(&frame->hold); + } + if (frame->twins_live) + { + frame->twins_live = false; + txn_spill_taken_drop(&frame->twins); + } + if (frame->decoded_live) + { + frame->decoded_live = false; + txn_runs_Decoded_drop(&frame->decoded); + } + if (frame->bytes_live) + { + frame->bytes_live = false; + std_vec_free_Checked(&frame->bytes, 1, 1); + } + if (frame->name_live) + { + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + } + if (frame->path_live) + { + frame->path_live = false; + std_vec_free_Checked(&frame->path.bytes, 1, 1); + } +} + +/* txn_spill_settled_from_read reads a spill's file of the pending names whole into a buffer, with its xxh3, its filter's bits, and its length. */ +static bool +txn_spill_settled_from_read(txn_spill_settled_from_frame_0_t *frame, const TxnRunsNames *names, StdVec *bytes, + bool *live, bool bits, TxnSpillFile *result) +{ + ChunksErrorError failed; + + txn_runs_spill_name(names, frame->pended->number, &frame->name); + frame->name_live = true; + if (frame->path_live) + std_PathBuf_set_file_name(&frame->path, (const char *) frame->name.ptr, frame->name.len); + else + { + txn_runs_spill_directory(frame->pended->storage.database, frame->home->own(frame->record), &frame->path); + frame->path_live = true; + { + StdPathBuf joined; + + std_path_join(frame->path.bytes.ptr, frame->path.bytes.len, (const char *) frame->name.ptr, + frame->name.len, &joined); + std_vec_free_Checked(&frame->path.bytes, 1, 1); + frame->path = joined; + } + } + frame->name_live = false; + std_vec_free_Checked(&frame->name, 1, 1); + if (!chunks_file_read_file(HOST_IO_IO_READ_DOORS(frame->io), frame->path.bytes.ptr, frame->path.bytes.len, bytes, + &failed)) + { + *frame->error = txn_base_storage(&failed); + return false; + } + *live = true; + *result = (TxnSpillFile) + { + .xid = frame->pended->xid, .kind = names->kind, .number = frame->pended->number, .len = (uint64) bytes->len + }; + return txn_runs_spill_hash(bytes->ptr, bytes->len, &result->hash, frame->error) && + (!bits || txn_runs_spill_bits(bytes->ptr, bytes->len, &result->bits, frame->error)); +} + +/* txn_spill_settled_from_twins gathers each version of a key the file holds whose record the memtable's chain holds too, by its bytes, and takes them out. */ +static bool +txn_spill_settled_from_twins(txn_spill_settled_from_frame_0_t *frame, TxnWrittenOf of) +{ + std_vec_new(&frame->twins, sizeof(txn_TupleVecU8VecHandle_t), _Alignof(txn_TupleVecU8VecHandle_t)); + frame->twins_live = true; + if (!host_delta_hold(frame->io, &frame->hold, frame->error)) + return false; + frame->hold_live = true; + for (size_t k = 0; k < frame->decoded.keys.len; k++) + { + const TxnRunsKey *key = &STD_VEC_AT(&frame->decoded.keys, TxnRunsKey, k); + txn_TupleVecU8VecHandle_t twin; + + if (!txn_spill_Home_chain(frame->home, frame->io, &frame->hold, frame->record, of, key->key.ptr, key->key.len, + &frame->chain, frame->error)) + return false; + frame->chain_live = true; + std_vec_new(&frame->handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + frame->handles_live = true; + for (size_t c = 0; c < frame->chain.len; c++) + { + const TxnMemtableVersionVersion *version = &STD_VEC_AT(&frame->chain, TxnMemtableVersionVersion, c); + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); + StdBytes bytes = txn_memtable_version_Record_bytes(&record); + bool twinned = false; + + for (size_t v = 0; v < key->versions.len && !twinned; v++) + { + StdBytes kept = txn_memtable_version_Record_bytes(&STD_VEC_AT(&key->versions, + TxnMemtableVersionRecord, v)); + + twinned = kept.len == bytes.len && memcmp(kept.ptr, bytes.ptr, bytes.len) == 0; + } + if (twinned) + std_vec_push_Checked(&frame->handles, &version->handle, sizeof(version->handle), + _Alignof(TxnMemtableVersionHandle)); + } + frame->chain_live = false; + txn_memtable_version_versions_drop(&frame->chain, 0); + if (frame->handles.len == 0) + { + frame->handles_live = false; + std_vec_free_Checked(&frame->handles, sizeof(TxnMemtableVersionHandle), + _Alignof(TxnMemtableVersionHandle)); + continue; + } + if (!std_vec_try_reserve(&frame->twins, 1, sizeof(twin), _Alignof(txn_TupleVecU8VecHandle_t))) + { + *frame->error = host_Error_out_of_memory(); + return false; + } + std_slice_to_vec_Checked(&twin.f0, key->key.ptr, key->key.len, 1, 1); + twin.f1 = frame->handles; + frame->handles_live = false; + std_vec_push_Checked(&frame->twins, &twin, sizeof(twin), _Alignof(txn_TupleVecU8VecHandle_t)); + } + frame->hold_live = false; + host_delta_Hold_drop(&frame->hold); + return txn_spill_removed_keys(frame->home, frame->io, frame->record, of, &frame->twins, frame->error); +} + +/* txn_spill_settled_from_body_0 is settled_from's body. */ +static void +txn_spill_settled_from_body_0(void *frame_) +{ + txn_spill_settled_from_frame_0_t *frame = frame_; + const TxnSpillPended *pended = frame->pended; + TxnRunsNames names; + TxnSpillFile listing[2] = {0}; + size_t listing_len = 1; + TxnWrittenOf of; + txn_TupleU64KindU32OptionU32_t replacing = {pended->xid, pended->kind, pended->number, true, pended->sources}; + + if (!txn_spill_names(frame->home, frame->record, &pended->storage, pended->kind, pended->xid, &names, + frame->error) || + !txn_spill_settled_from_read(frame, &names, &frame->bytes, &frame->bytes_live, true, &listing[0]) || + !txn_runs_decode(frame->bytes.ptr, frame->bytes.len, &names, listing[0].hash, &frame->decoded, frame->error)) + { + txn_spill_settled_from_pad_0(frame); + return; + } + frame->decoded_live = true; + switch (pended->kind) + { + case TXN_RUNS_KIND_SPILLED_ROWS: + case TXN_RUNS_KIND_SPILLED_ENTRIES: + of = TXN_WRITTEN_OF_ROWS; + break; + case TXN_RUNS_KIND_SPILLED_VALUES: + of = TXN_WRITTEN_OF_VALUES; + break; + case TXN_RUNS_KIND_SPILLED_FREE: + of = TXN_WRITTEN_OF_FREE; + break; + case TXN_RUNS_KIND_SPILLED_CHUNKS: + of = TXN_WRITTEN_OF_CHUNKS; + break; + case TXN_RUNS_KIND_SPILLED_IDENTITY: + of = TXN_WRITTEN_OF_IDENTITY; + break; + default: + *frame->error = txn_spill_internal("a process's slot names a spill of no tree"); + txn_spill_settled_from_pad_0(frame); + return; + } + if (!pended->has_sources && !txn_spill_settled_from_twins(frame, of)) + { + txn_spill_settled_from_pad_0(frame); + return; + } + if (pended->tupled) + { + TxnRunsNames tuple_names; + + if (!txn_spill_names(frame->home, frame->record, &pended->storage, TXN_RUNS_KIND_SPILLED_TUPLES, pended->xid, + &tuple_names, frame->error) || + !txn_spill_settled_from_read(frame, &tuple_names, &frame->tuple_bytes, &frame->tuple_bytes_live, false, + &listing[1])) + { + txn_spill_settled_from_pad_0(frame); + return; + } + if (!pended->has_sources) + { + if (!txn_runs_decode_tuples(frame->tuple_bytes.ptr, frame->tuple_bytes.len, &tuple_names, listing[1].hash, + &frame->rows, frame->error)) + { + txn_spill_settled_from_pad_0(frame); + return; + } + frame->rows_live = true; + std_vec_with_capacity_Checked(&frame->tids, frame->rows.len, sizeof(uint64), _Alignof(uint64)); + frame->tids_live = true; + for (size_t k = 0; k < frame->rows.len; k++) + std_vec_push_Checked(&frame->tids, &STD_VEC_AT(&frame->rows, txn_TupleU64RefSliceU8_t, k).f0, + sizeof(uint64), _Alignof(uint64)); + if (!frame->home->untupled(frame->io, frame->record, frame->tids.ptr, frame->tids.len, frame->error)) + { + txn_spill_settled_from_pad_0(frame); + return; + } + } + if (!txn_runs_spill_bits(frame->tuple_bytes.ptr, frame->tuple_bytes.len, &listing[1].bits, frame->error)) + { + txn_spill_settled_from_pad_0(frame); + return; + } + listing_len = 2; + } + frame->ok = txn_spill_linked(frame->home, frame->io, frame->record, &pended->storage, listing, listing_len, + pended->has_sources, replacing, frame->error) && + txn_spill_settled(frame->home, frame->io, &frame->record, frame->error); + txn_spill_settled_from_pad_0(frame); +} + +/* txn_spill_settled_from lists a pending spill in its home's manifest as settle_pending does. */ +static bool +txn_spill_settled_from(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, + const TxnSpillPended *pended, HostError *error) +{ + txn_spill_settled_from_frame_0_t frame = {.home = home, .io = io, .record = record, .pended = pended, + .error = error}; + + std_landing(&frame, txn_spill_settled_from_body_0, txn_spill_settled_from_pad_0); + return frame.ok; +} + +/* txn_spill_settle_pending_captures_t is settle_pending's step's captures: the backend's I/O and its pending spill. */ +typedef struct txn_spill_settle_pending_captures_t +{ + HostIoIo *io; + const TxnSpillPended *pended; +} txn_spill_settle_pending_captures_t; + +/* txn_spill_settle_pending_closure_0 is settle_pending's step: the pending spill listed in its home's manifest. */ +static bool +txn_spill_settle_pending_closure_0(void *captures, const TxnSpillHomeOps *home, const HostSharedHeld *held, + HostError *error) +{ + txn_spill_settle_pending_captures_t *c = captures; + + return txn_spill_settled_from(home, c->io, host_shared_Held_record(held), c->pended, error); +} + +/* + * txn_spill_settle_pending lists the spill the process's slot names, read + * back from its file, and clears the slot; a slot whose storage has no state + * any more is cleared alone. It first takes out of the memtable each version + * of a key the file holds whose record is one the file holds, by its bytes, + * and drops the tids of the spill's tuple-id file, then lists the spill's + * tuple-id file after its row file, and a merge's files in place of those it + * replaces. A bootstrap process has no slot. + */ +bool +txn_spill_settle_pending(HostIoIo *io, HostError *error) +{ + bool some; + TxnSpillPended pended; + txn_spill_settle_pending_captures_t captures = {io, &pended}; + + if (!host_shared_ready()) + return true; + if (!txn_spill_pending(io, &some, &pended, error)) + return false; + if (!some) + return true; + if (!txn_spill_at_found(io, &pended.storage, txn_spill_settle_pending_closure_0, &captures, &some, error)) + return false; + return some || txn_spill_settled(&TXN_SPILL_STATE_HOME_OPS, io, NULL, error); +} + +/* + * txn_spill_take_settled settles the spill the process's slot names, as + * settle_pending does, when an error or an exit stopped its take-out of + * versions from the memtable, so the take-out ends and readers wait no + * longer. + */ +bool +txn_spill_take_settled(HostIoIo *io, HostError *error) +{ + HostSharedMine mine; + + if (!host_shared_ready()) + return true; + if (!txn_spill_mine(io, &mine, error)) + return false; + return host_shared_Mine_load(&mine, TXN_SPILL_PENDING_TAKING) == 0 || txn_spill_settle_pending(io, error); +} + +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/simc/simc.h" +#include "doltgres/std/fmt.h" +#include "doltgres/std/test.h" +#include "doltgres/txn/memtable/testing.h" + +/* TXN_SPILL_TESTS_KIND_NAMES are Kind's variants' names, at their numbers, for its derived Debug. */ +static const char *const TXN_SPILL_TESTS_KIND_NAMES[] = {"Rows", "Free", "Values", "Chunks", "Status", "SpilledRows", +"SpilledValues", "SpilledTuples", "Entries", "SpilledEntries", "Identity", "SpilledFree", "SpilledChunks"}; + +/* txn_spill_tests_Kind_Debug_fmt is Kind's derived Debug, its variant's name, for the tests' asserts. */ +static bool +txn_spill_tests_Kind_Debug_fmt(const void *value, StdFormatter *f) +{ + const char *name = TXN_SPILL_TESTS_KIND_NAMES[*(const TxnRunsKind *) value]; + + return std_fmt_write_str(f, name, strlen(name)); +} + +/* txn_spill_tests_File_Debug_fmt is File's derived Debug, File { xid, kind, number, len, hash, bits }, for the tests' asserts. */ +static bool +txn_spill_tests_File_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnSpillFile *file = value; + StdDebugStruct fields = std_fmt_debug_struct(f, "File", 4); + + std_DebugStruct_field(&fields, "xid", 3, &file->xid, std_u64_Debug_fmt); + std_DebugStruct_field(&fields, "kind", 4, &file->kind, txn_spill_tests_Kind_Debug_fmt); + std_DebugStruct_field(&fields, "number", 6, &file->number, std_u32_Debug_fmt); + std_DebugStruct_field(&fields, "len", 3, &file->len, std_u64_Debug_fmt); + std_DebugStruct_field(&fields, "hash", 4, &file->hash, std_u64_Debug_fmt); + std_DebugStruct_field(&fields, "bits", 4, &file->bits, std_u64_Debug_fmt); + return std_DebugStruct_finish(&fields); +} + +/* txn_spill_tests_files_Debug_fmt is a Vec's Debug. */ +static bool +txn_spill_tests_files_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *files = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < files->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(files, TxnSpillFile, k), txn_spill_tests_File_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_spill_tests_files_are asserts at its place that a Vec is a list of files, and frees the Vec. */ +static void +txn_spill_tests_files_are(uint32 line, uint32 col, StdVec *files, const TxnSpillFile *wanted, size_t wanted_len) +{ + StdVec wanted_vec = {(void *) wanted, wanted_len, wanted_len}; + bool eq = files->len == wanted_len; + + for (size_t k = 0; eq && k < wanted_len; k++) + eq = txn_spill_File_PartialEq_eq(&STD_VEC_AT(files, TxnSpillFile, k), &wanted[k]); + STD_ASSERT_EQ(line, col, eq, files, txn_spill_tests_files_Debug_fmt, &wanted_vec, txn_spill_tests_files_Debug_fmt); + std_vec_free_Checked(files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); +} + +/* txn_spill_tests_Slotted_Debug_fmt is Slotted's Debug, the tuple (xid, kind, number, sources). */ +static bool +txn_spill_tests_Slotted_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_TupleU64KindU32OptionU32_t *slotted = value; + StdOptionRef sources = {slotted->has_f3, STD_ITEM(&slotted->f3, std_u32_Debug_fmt)}; + StdTupleRef tuple = STD_TUPLE(4, STD_ITEM(&slotted->f0, std_u64_Debug_fmt), + STD_ITEM(&slotted->f1, txn_spill_tests_Kind_Debug_fmt), + STD_ITEM(&slotted->f2, std_u32_Debug_fmt), + STD_ITEM(&sources, std_option_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_spill_tests_slotteds_Debug_fmt is a Vec's Debug. */ +static bool +txn_spill_tests_slotteds_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *slotteds = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < slotteds->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(slotteds, txn_TupleU64KindU32OptionU32_t, k), + txn_spill_tests_Slotted_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_spill_tests_Pended_Debug_fmt is Pended's Debug, the tuple (storage, xid, kind, number, tupled, sources). */ +static bool +txn_spill_tests_Pended_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnSpillPended *pended = value; + StdOptionRef sources = {pended->has_sources, STD_ITEM(&pended->sources, std_u32_Debug_fmt)}; + StdTupleRef tuple = STD_TUPLE(6, STD_ITEM(&pended->storage, host_Storage_Debug_fmt), + STD_ITEM(&pended->xid, std_u64_Debug_fmt), + STD_ITEM(&pended->kind, txn_spill_tests_Kind_Debug_fmt), + STD_ITEM(&pended->number, std_u32_Debug_fmt), + STD_ITEM(&pended->tupled, std_bool_Debug_fmt), + STD_ITEM(&sources, std_option_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_spill_tests_file returns a test's spill file of rows by a transaction, of a number, which no file on disk holds. */ +static TxnSpillFile +txn_spill_tests_file(uint64 xid, uint32 number) +{ + return (TxnSpillFile) + { + .xid = xid, .kind = TXN_RUNS_KIND_SPILLED_ROWS, .number = number, .len = 100 + (uint64) number, + .hash = xid ^ 0xfeed, .bits = 64 + }; +} + +/* txn_spill_tests_files_of returns the files a state's storage's manifest lists, newest first. */ +static StdVec +txn_spill_tests_files_of(HostIoIo *io, HostSharedRecord state) +{ + HostStorage storage; + TxnStatePersistence persistence; + txn_TupleVecFileU64_t listed; + HostError error; + + if (!txn_state_described(state, &storage, &persistence)) + STD_OPTION_EXPECT_FAILED(1854, 52, "the state is filled"); + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &listed, &error)) + STD_RESULT_EXPECT_FAILED(1855, 37, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return listed.f0; +} + +/* + * txn_spill_tests_written writes a test's spill file of rows of a storage's + * state by a transaction, of a number, holding its keys with no versions, and + * its filter, and returns it with its path. + */ +static TxnSpillFile +txn_spill_tests_written(HostIoIo *io, HostSharedRecord state, uint64 xid, uint32 number, const StdBytes *keys, + size_t keys_len, StdPathBuf *path) +{ + TxnRunsTail tail = {.tag = TXN_RUNS_TAIL_NONE}; + HostStorage storage; + TxnStatePersistence persistence; + StdPathBuf dir; + StdStr made[2]; + ChunksErrorError failed; + TxnRunsNames names; + TxnRunsWriting *writing; + TxnSpillFilter filter; + HostError error; + RepoRunFile run; + StdString name; + TxnSpillFile file; + bool some; + + if (!txn_state_described(state, &storage, &persistence)) + STD_OPTION_EXPECT_FAILED(1866, 52, "the state is filled"); + txn_runs_spill_directory(storage.database, txn_state_own(state), &dir); + if (!std_path_parent(dir.bytes.ptr, dir.bytes.len, &made[0])) + STD_OPTION_UNWRAP_FAILED(1868, 35); + made[1] = (StdStr) + { + dir.bytes.ptr, dir.bytes.len + }; + for (size_t k = 0; k < lengthof(made); k++) + if (!chunks_file_create_dir(HOST_IO_IO_IO_DOORS(io), made[k].ptr, made[k].len, &failed)) + STD_RESULT_EXPECT_FAILED(1869, 48, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, state, &storage, TXN_RUNS_KIND_SPILLED_ROWS, xid, &names, &error)) + STD_RESULT_EXPECT_FAILED(1871, 68, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + writing = txn_runs_Writing_spill(dir.bytes.ptr, dir.bytes.len, &names, tail, number); + for (size_t k = 0; k < keys_len; k++) + if (!txn_runs_Writing_key(writing, HOST_IO_IO_IO_DOORS(io), keys[k].ptr, keys[k].len, NULL, 0, &error)) + STD_RESULT_EXPECT_FAILED(1874, 39, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_spill_Filter_of(keys_len, keys, keys_len, &filter, &error)) + STD_RESULT_EXPECT_FAILED(1876, 67, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_runs_Writing_end(writing, HOST_IO_IO_IO_DOORS(io), filter.words.ptr, filter.words.len, &some, &run, &error)) + STD_RESULT_EXPECT_FAILED(1877, 50, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!some) + STD_OPTION_UNWRAP_FAILED(1877, 59); + txn_runs_Writing_drop(writing); + file = (TxnSpillFile) + { + .xid = xid, .kind = TXN_RUNS_KIND_SPILLED_ROWS, .number = number, .len = run.len, + .hash = run.hash, .bits = txn_spill_Filter_bits(&filter) + }; + txn_runs_spill_name(&names, number, &name); + std_path_join(dir.bytes.ptr, dir.bytes.len, name.ptr, name.len, path); + std_String_drop(&name); + std_vec_free_Checked(&filter.words, sizeof(uint64), _Alignof(uint64)); + std_vec_free_Checked(&dir.bytes, 1, 1); + return file; +} + +/* txn_spill_tests_of_742 is the list, hit, and removal case's first removal's pick: transaction 742's files. */ +static bool +txn_spill_tests_of_742(void *captures, const TxnSpillFile *file) +{ + return file->xid == 742; +} + +/* txn_spill_tests_every is the list, hit, and removal case's second removal's pick: every file. */ +static bool +txn_spill_tests_every(void *captures, const TxnSpillFile *file) +{ + return true; +} + +/* + * Bugs row 143's manifest: listed spills list newest first with what names + * their files, each file's filter, read from its own pages, hits its own keys + * and misses most others, and a removal takes exactly the files it picks out + * of the manifest, the delta's used bytes unmoved by any of it. + */ +void +test_txn_spill_tests_a_storages_spills_list_hit_and_remove(void) +{ + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0), STD_FMT_LIT(" misses")}; + static const StdBytes first[] = {{(const uint8 *) "a", 1}, {(const uint8 *) "b", 1}}; + static const StdBytes second[] = {{(const uint8 *) "c", 1}}; + static const StdBytes third[] = {{(const uint8 *) "d", 1}}; + static const StdBytes *const keys[] = {first, second, third}; + static const size_t keys_lens[] = {lengthof(first), lengthof(second), lengthof(third)}; + static const uint64 xids[] = {742, 742, 800}; + static const uint32 numbers[] = {0, 1, 0}; + txn_TupleU64KindU32OptionU32_t no_replacing = {0}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9741); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9741); + uint64 used = host_delta_used(); + uint64 now; + TxnSpillFile spills[3]; + StdPathBuf paths[3]; + TxnSpillFile wanted[3]; + StdVec files; + StdVec gone; + StdPathBuf manifest; + HostError error; + HostFile opened; + bool there; + bool hit; + + for (size_t k = 0; k < lengthof(spills); k++) + spills[k] = txn_spill_tests_written(io, state, xids[k], numbers[k], keys[k], keys_lens[k], &paths[k]); + for (size_t k = 0; k < lengthof(spills); k++) + if (!txn_spill_linked(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &spills[k], 1, false, no_replacing, + &error)) + STD_RESULT_EXPECT_FAILED(1899, 63, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + now = host_delta_used(); + STD_ASSERT_EQ_MSG(1901, 9, now == used, &now, std_u64_Debug_fmt, &used, std_u64_Debug_fmt, + "the manifest takes none of the delta"); + files = txn_spill_tests_files_of(io, state); + wanted[0] = spills[2]; + wanted[1] = spills[1]; + wanted[2] = spills[0]; + txn_spill_tests_files_are(1902, 9, &files, wanted, 3); + for (size_t k = 0; k < lengthof(spills); k++) + { + size_t misses = 0; + + hit = true; + for (size_t j = 0; hit && j < keys_lens[k]; j++) + if (!txn_spill_hits(io, &paths[k], &spills[k], keys[k][j].ptr, keys[k][j].len, &hit, &error)) + STD_RESULT_EXPECT_FAILED(1904, 71, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(1904, 13, hit, "keys.iter().all(|key| hits(io, (path, file), key).unwrap())"); + for (uint32 other = 0; other < 100; other++) + { + uint8 key[4]; + + std_u32_to_be_bytes(other, key); + if (!txn_spill_hits(io, &paths[k], &spills[k], key, sizeof(key), &hit, &error)) + STD_RESULT_EXPECT_FAILED(1905, 99, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!hit) + misses++; + } + if (misses < 90) + { + StdFmtArg args[1] = {{&misses, std_usize_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_panic_fmt(RUST_FILE, 1906, 13, &message); + } + } + if (!txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, txn_spill_tests_of_742, NULL, &gone, + &error)) + STD_RESULT_EXPECT_FAILED(1908, 79, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + wanted[0] = spills[0]; + wanted[1] = spills[1]; + txn_spill_tests_files_are(1909, 9, &gone, wanted, 2); + files = txn_spill_tests_files_of(io, state); + txn_spill_tests_files_are(1910, 9, &files, &spills[2], 1); + if (!txn_spill_removed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, txn_spill_tests_every, NULL, &gone, + &error)) + STD_RESULT_EXPECT_FAILED(1911, 54, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&gone, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + files = txn_spill_tests_files_of(io, state); + STD_ASSERT(1912, 9, files.len == 0, "files_of(io, state).is_empty()"); + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + txn_spill_manifest(&TXN_SPILL_STATE_HOME_OPS, state, &storage, &manifest); + if (!host_open_if_there(HOST_IO_IO_READ_DOORS(io), manifest.bytes.ptr, manifest.bytes.len, &there, &opened, + &error)) + STD_RESULT_EXPECT_FAILED(1913, 75, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (there) + host_File_drop(&opened); + std_vec_free_Checked(&manifest.bytes, 1, 1); + STD_ASSERT(1913, 9, !there, "host::open_if_there(io, &manifest(state, &storage(9741))).unwrap().is_none()"); + for (size_t k = 0; k < lengthof(paths); k++) + std_vec_free_Checked(&paths[k].bytes, 1, 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* txn_spill_tests_pended_are asserts at its line that a storage's pended spills are only a wanted one, or are none with an assert!'s message. */ +static void +txn_spill_tests_pended_are(uint32 line, uint32 unwrap_col, HostSharedRecord state, const HostStorage *storage, + const txn_TupleU64KindU32OptionU32_t *wanted, const char *empty) +{ + StdVec wanted_vec = {(void *) wanted, wanted != NULL, wanted != NULL}; + StdVec pended; + HostError error; + bool eq; + + if (!txn_spill_pended(&TXN_SPILL_STATE_HOME_OPS, state, storage, &pended, &error)) + STD_RESULT_EXPECT_FAILED(line, unwrap_col, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + eq = pended.len == wanted_vec.len; + if (eq && wanted != NULL) + { + const txn_TupleU64KindU32OptionU32_t *slotted = pended.ptr; + + eq = slotted->f0 == wanted->f0 && slotted->f1 == wanted->f1 && slotted->f2 == wanted->f2 && + slotted->has_f3 == wanted->has_f3 && (!slotted->has_f3 || slotted->f3 == wanted->f3); + } + if (!eq && wanted == NULL) + std_panic(RUST_FILE, line, 9, empty, strlen(empty)); + STD_ASSERT_EQ(line, 9, eq, &pended, txn_spill_tests_slotteds_Debug_fmt, &wanted_vec, + txn_spill_tests_slotteds_Debug_fmt); + std_vec_free_Checked(&pended, sizeof(txn_TupleU64KindU32OptionU32_t), _Alignof(txn_TupleU64KindU32OptionU32_t)); +} + +/* txn_spill_tests_pending_is asserts at its place that the process's slot names a pended spill, or none. */ +static void +txn_spill_tests_pending_is(uint32 line, HostIoIo *io, const TxnSpillPended *wanted) +{ + TxnSpillPended pended; + StdOptionRef left; + StdOptionRef right = {wanted != NULL, STD_ITEM(wanted, txn_spill_tests_Pended_Debug_fmt)}; + HostError error; + bool some; + bool eq; + + if (!txn_spill_pending(io, &some, &pended, &error)) + STD_RESULT_EXPECT_FAILED(line, 32, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + left = (StdOptionRef) + { + some, STD_ITEM(&pended, txn_spill_tests_Pended_Debug_fmt) + }; + eq = some == (wanted != NULL); + if (eq && some) + eq = host_Storage_PartialEq_eq(&pended.storage, &wanted->storage) && pended.xid == wanted->xid && + pended.kind == wanted->kind && pended.number == wanted->number && pended.tupled == wanted->tupled && + pended.has_sources == wanted->has_sources && (!pended.has_sources || pended.sources == wanted->sources); + STD_ASSERT_EQ(line, 9, eq, &left, std_option_Debug_fmt, &right, std_option_Debug_fmt); +} + +/* + * Bugs row 74's pending slot: a process's pending spill is found by a reader + * of its storage, not of another, while it is pending, and a second pend + * before it settles is refused. + */ +void +test_txn_spill_tests_a_pending_spill_is_found_by_its_storages_readers_until_it_settles(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9742); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9742); + HostStorage other = storage; + txn_TupleU64KindU32OptionU32_t slotted = {.f0 = 742, .f1 = TXN_RUNS_KIND_SPILLED_ROWS, .f2 = 3}; + TxnSpillPended pended = {.storage = storage, .xid = 742, .kind = TXN_RUNS_KIND_SPILLED_ROWS, .number = 3}; + TxnSpillFile file; + HostError error; + bool refused; + + other.relation = 16385; + txn_spill_tests_pended_are(1925, 41, state, &storage, NULL, "assertion failed: pended(state, &storage).unwrap().is_empty()"); + file = txn_spill_tests_file(742, 3); + if (!txn_spill_pend(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &file, false, false, 0, &error)) + STD_RESULT_EXPECT_FAILED(1926, 65, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_spill_tests_pended_are(1927, 44, state, &storage, &slotted, NULL); + txn_spill_tests_pended_are(1928, 39, state, &other, NULL, + "assertion failed: pended(state, &other).unwrap().is_empty()"); + txn_spill_tests_pending_is(1929, io, &pended); + file = txn_spill_tests_file(742, 4); + refused = !txn_spill_pend(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &file, false, false, 0, &error); + if (refused) + host_Error_drop(&error); + STD_ASSERT(1930, 9, refused, "pend(io, state, &storage, (&file(742, 4), false), None).is_err()"); + if (!txn_spill_settled(&TXN_SPILL_STATE_HOME_OPS, io, &state, &error)) + STD_RESULT_EXPECT_FAILED(1931, 34, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_spill_tests_pended_are(1932, 41, state, &storage, NULL, "assertion failed: pended(state, &storage).unwrap().is_empty()"); + txn_spill_tests_pending_is(1933, io, NULL); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* txn_spill_tests_numbered returns the four-byte big-endian keys of a count of numbers from zero, and their slices. */ +static void +txn_spill_tests_numbered(uint32 count, StdVec *bytes, StdVec *keys) +{ + std_vec_from_elem_zeroed(bytes, count, 4, 1); + std_vec_from_elem_zeroed(keys, count, sizeof(StdBytes), _Alignof(StdBytes)); + for (uint32 key = 0; key < count; key++) + { + uint8 *at = (uint8 *) bytes->ptr + 4 * (size_t) key; + + std_u32_to_be_bytes(key, at); + STD_VEC_AT(keys, StdBytes, key) = (StdBytes) + { + at, 4 + }; + } +} + +/* + * Bugs row 74's filter: every key a filter was built of hits it, and of a + * thousand other keys about one in a hundred does, at most three in a + * hundred. + */ +void +test_txn_spill_tests_a_filter_hits_its_keys_and_few_others(void) +{ + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0), STD_FMT_LIT(" false hits")}; + StdVec bytes; + StdVec keys; + TxnSpillFilter filter; + HostError error; + uint64 bits; + uint64 wanted = 10048; + size_t false_hits = 0; + bool all = true; + + txn_spill_tests_numbered(1000, &bytes, &keys); + if (!txn_spill_Filter_of(keys.len, keys.ptr, keys.len, &filter, &error)) + STD_RESULT_EXPECT_FAILED(1941, 78, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + bits = txn_spill_Filter_bits(&filter); + STD_ASSERT_EQ(1942, 9, bits == wanted, &bits, std_u64_Debug_fmt, &wanted, std_u64_Debug_fmt); + for (size_t k = 0; all && k < keys.len; k++) + all = txn_spill_Filter_hits(&filter, STD_VEC_AT(&keys, StdBytes, k).ptr, 4); + STD_ASSERT(1943, 9, all, "keys.iter().all(|key| filter.hits(key))"); + for (uint32 key = 1000; key < 2000; key++) + { + uint8 be[4]; + + std_u32_to_be_bytes(key, be); + if (txn_spill_Filter_hits(&filter, be, sizeof(be))) + false_hits++; + } + if (false_hits > 30) + { + StdFmtArg args[1] = {{&false_hits, std_usize_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_panic_fmt(RUST_FILE, 1945, 9, &message); + } + std_vec_free_Checked(&filter.words, sizeof(uint64), _Alignof(uint64)); + std_vec_free_Checked(&keys, sizeof(StdBytes), _Alignof(StdBytes)); + std_vec_free_Checked(&bytes, 4, 1); +} + +/* + * Bugs row 143's filter pages: a file's filter past a page's bits is written + * in several pages, each of the most words but the last, which the file's + * trailer places, and a key whose bits fall in each page hits. + */ +void +test_txn_spill_tests_a_large_filter_takes_several_pages(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9517); + HostSharedRecord state = host_shared_Held_record(&held); + StdVec bytes; + StdVec listed; + StdPathBuf path; + TxnSpillFile file; + HostFile opened; + ChunksErrorError failed; + HostError error; + StdVec page; + size_t pages[3]; + size_t wanted[3]; + StdVec pages_vec = {pages, 3, 3}; + StdVec wanted_vec = {wanted, 3, 3}; + uint64 count; + uint64 three = 3; + bool refused; + bool hit = true; + + txn_spill_tests_numbered(14000, &bytes, &listed); + file = txn_spill_tests_written(io, state, 744, 0, listed.ptr, listed.len, &path); + count = std_div_ceil_u64(file.bits, TXN_SPILL_PAGE_BITS, RUST_FILE, 1959, 30); + STD_ASSERT_EQ(1959, 9, count == three, &count, std_u64_Debug_fmt, &three, std_u64_Debug_fmt); + if (!chunks_file_open_rw(HOST_IO_IO_IO_DOORS(io), path.bytes.ptr, path.bytes.len, false, &opened, &failed)) + STD_RESULT_EXPECT_FAILED(1960, 62, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + for (uint64 at = 0; at < 3; at++) + { + if (!txn_runs_spill_filter(io, &opened, file.len, file.bits, at, &page, &error)) + STD_RESULT_EXPECT_FAILED(1962, 92, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + pages[at] = page.len; + std_vec_free_Checked(&page, 1, 1); + } + wanted[0] = TXN_RUNS_FILTER_WORDS * 8; + wanted[1] = TXN_RUNS_FILTER_WORDS * 8; + wanted[2] = (size_t) (file.bits - 2 * TXN_SPILL_PAGE_BITS) / 8; + STD_ASSERT_EQ(1963, 9, pages[0] == wanted[0] && pages[1] == wanted[1] && pages[2] == wanted[2], &pages_vec, + txn_memtable_table_tests_usizes_Debug_fmt, &wanted_vec, txn_memtable_table_tests_usizes_Debug_fmt); + refused = !txn_runs_spill_filter(io, &opened, file.len, file.bits, 3, &page, &error); + if (refused) + host_Error_drop(&error); + else + std_vec_free_Checked(&page, 1, 1); + STD_ASSERT(1964, 9, refused, "runs::spill_filter(io, &opened, (file.len, file.bits), 3).is_err()"); + for (size_t k = 0; hit && k < listed.len; k++) + if (!txn_spill_hits(io, &path, &file, STD_VEC_AT(&listed, StdBytes, k).ptr, 4, &hit, &error)) + STD_RESULT_EXPECT_FAILED(1965, 71, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(1965, 9, hit, "listed.iter().all(|key| hits(io, (&path, &file), key).unwrap())"); + host_File_drop(&opened); + std_vec_free_Checked(&path.bytes, 1, 1); + std_vec_free_Checked(&listed, sizeof(StdBytes), _Alignof(StdBytes)); + std_vec_free_Checked(&bytes, 4, 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* txn_spill_tests_read_t is a spill case's read key: its key and its versions' values, newest first. */ +typedef struct txn_spill_tests_read_t +{ + StdBytes key; + /* values is a Vec>. */ + StdVec values; +} txn_spill_tests_read_t; + +/* txn_spill_tests_read_Debug_fmt is a read key's Debug, the tuple (key, values). */ +static bool +txn_spill_tests_read_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_spill_tests_read_t *read = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&read->key, std_bytes_Debug_fmt), + STD_ITEM(&read->values, txn_memtable_table_tests_keys_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_spill_tests_reads_Debug_fmt is a Vec of read keys' Debug. */ +static bool +txn_spill_tests_reads_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *reads = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < reads->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(reads, txn_spill_tests_read_t, k), txn_spill_tests_read_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_spill_tests_reads_eq reports whether two Vecs of read keys are equal. */ +static bool +txn_spill_tests_reads_eq(const StdVec *a, const StdVec *b) +{ + if (a->len != b->len) + return false; + for (size_t k = 0; k < a->len; k++) + { + const txn_spill_tests_read_t *x = &STD_VEC_AT(a, txn_spill_tests_read_t, k); + const txn_spill_tests_read_t *y = &STD_VEC_AT(b, txn_spill_tests_read_t, k); + + if (x->key.len != y->key.len || memcmp(x->key.ptr, y->key.ptr, x->key.len) != 0 || + !txn_memtable_table_tests_keys_eq(&x->values, &y->values)) + return false; + } + return true; +} + +/* txn_spill_tests_reads_free frees a Vec of read keys and their values. */ +static void +txn_spill_tests_reads_free(StdVec *reads) +{ + for (size_t k = 0; k < reads->len; k++) + txn_memtable_table_tests_keys_free(&STD_VEC_AT(reads, txn_spill_tests_read_t, k).values); + std_vec_free_Checked(reads, sizeof(txn_spill_tests_read_t), _Alignof(txn_spill_tests_read_t)); +} + +/* txn_spill_tests_value returns a version's value, as a spill case's match on its owned op reads it: a put's value, or empty for any other op. */ +static StdVec +txn_spill_tests_value(const TxnMemtableVersionRecord *record) +{ + TxnMemtableVersionOpRef op = txn_memtable_version_Record_op(record); + TxnMemtableOp owned = txn_memtable_version_OpRef_owned(&op); + StdVec value; + + if (owned.tag != TXN_MEMTABLE_OP_PUT) + { + txn_memtable_Op_drop(&owned); + std_vec_new(&value, 1, 1); + return value; + } + value = owned.u.put.value; + std_vec_new(&owned.u.put.value, 1, 1); + txn_memtable_Op_drop(&owned); + return value; +} + +/* txn_spill_tests_versioned_t is a spill case's chain entry: a version's writer's xid and its value. */ +typedef struct txn_spill_tests_versioned_t +{ + uint64 xid; + /* value is a Vec. */ + StdVec value; +} txn_spill_tests_versioned_t; + +/* txn_spill_tests_versioned_Debug_fmt is a chain entry's Debug, the tuple (xid, value). */ +static bool +txn_spill_tests_versioned_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_spill_tests_versioned_t *versioned = value; + StdBytes bytes = {versioned->value.ptr, versioned->value.len}; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&versioned->xid, std_u64_Debug_fmt), + STD_ITEM(&bytes, std_bytes_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_spill_tests_chain_Debug_fmt is a Vec of chain entries' Debug. */ +static bool +txn_spill_tests_chain_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *chain = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < chain->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(chain, txn_spill_tests_versioned_t, k), + txn_spill_tests_versioned_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_spill_tests_chain_is asserts at its line that a key's chain is a list of xids and values, and frees the chain. */ +static void +txn_spill_tests_chain_is(uint32 line, StdVec *chain, const uint64 *xids, const char *const *values, size_t len) +{ + txn_spill_tests_versioned_t wanted[2]; + StdVec wanted_vec = {wanted, len, len}; + bool eq = chain->len == len; + + for (size_t k = 0; k < len; k++) + { + wanted[k].xid = xids[k]; + std_slice_to_vec_Checked(&wanted[k].value, (const uint8 *) values[k], strlen(values[k]), 1, 1); + } + for (size_t k = 0; eq && k < len; k++) + { + const txn_spill_tests_versioned_t *entry = &STD_VEC_AT(chain, txn_spill_tests_versioned_t, k); + + eq = entry->xid == wanted[k].xid && entry->value.len == wanted[k].value.len && + memcmp(entry->value.ptr, wanted[k].value.ptr, entry->value.len) == 0; + } + STD_ASSERT_EQ(line, 9, eq, chain, txn_spill_tests_chain_Debug_fmt, &wanted_vec, txn_spill_tests_chain_Debug_fmt); + for (size_t k = 0; k < len; k++) + std_vec_free_Checked(&wanted[k].value, 1, 1); + for (size_t k = 0; k < chain->len; k++) + std_vec_free_Checked(&STD_VEC_AT(chain, txn_spill_tests_versioned_t, k).value, 1, 1); + std_vec_free_Checked(chain, sizeof(txn_spill_tests_versioned_t), _Alignof(txn_spill_tests_versioned_t)); +} + +/* txn_spill_tests_chain returns a key's versions in a state's memtable and spills, each its writer's xid and its value, as the spill case's chain closure does. */ +static StdVec +txn_spill_tests_chain(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, const char *key) +{ + StdVec versions; + StdVec chain; + HostError error; + + if (!txn_memtable_table_versions(io, hold, state, (const uint8 *) key, strlen(key), &versions, &error)) STD_RESULT_EXPECT_FAILED(2035, 67, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - std_vec_with_capacity_Checked(&chain, versions.len, sizeof(txn_spill_tests_versioned_t), - _Alignof(txn_spill_tests_versioned_t)); - for (size_t k = 0; k < versions.len; k++) + std_vec_with_capacity_Checked(&chain, versions.len, sizeof(txn_spill_tests_versioned_t), + _Alignof(txn_spill_tests_versioned_t)); + for (size_t k = 0; k < versions.len; k++) + { + TxnMemtableVersionVersion *version = &STD_VEC_AT(&versions, TxnMemtableVersionVersion, k); + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); + txn_spill_tests_versioned_t entry = {.xid = txn_memtable_version_Record_writer(&record).xid, + .value = txn_spill_tests_value(&record)}; + + std_vec_push_Checked(&chain, &entry, sizeof(entry), _Alignof(txn_spill_tests_versioned_t)); + txn_memtable_version_Version_drop(version); + } + std_vec_free_Checked(&versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + return chain; +} + +/* + * Bugs row 74's spill: a spill takes out of the memtable exactly the keys + * whose chain holds only its transaction's versions, writes them, each key's + * versions newest first, to a synced file its entry names, and leaves a key + * another writer's version is in, whatever the order of the two. + */ +void +test_txn_spill_tests_a_spill_takes_only_the_keys_its_transaction_alone_holds(void) +{ + static const char *const left_keys[] = {"c", "d"}; + static const char *const a_values[] = {"1"}; + static const char *const b_values[] = {"3", "2"}; + static const char *const d_values[] = {"6", "5"}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9743); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9743); + uint64 mine = simc_begin().f0; + uint64 other = simc_begin().f0; + const struct + { + uint64 xid; + const char *key; + const char *value; + } pushes[] = {{mine, "a", "1"}, {mine, "b", "2"}, {mine, "b", "3"}, {other, "c", "4"}, {other, "d", "5"}, + {mine, "d", "6"}}; + uint64 b_xids[2] = {mine, mine}; + uint64 d_xids[2] = {mine, other}; + uint32 zero = 0; + uint64 sized[2]; + txn_spill_tests_read_t entries[2] = {{.key = {(const uint8 *) "a", 1}}, {.key = {(const uint8 *) "b", 1}}}; + StdTupleRef left_tuple; + StdTupleRef right_tuple; + StdVec left; + StdVec wanted; + StdVec listed; + StdVec bytes; + StdVec read; + StdVec chain; + StdPathBuf dir; + StdPathBuf path; + StdString name; + TxnRunsNames names; + TxnRunsDecoded decoded; + TxnSpillFile file; + HostDeltaHold hold; + ChunksErrorError failed; + HostError error; + size_t kept = 0; + size_t one = 1; + uint64 hash; + bool spilled; + bool hit; + + for (size_t k = 0; k < lengthof(pushes); k++) + { + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) pushes[k].value, 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + + if (!txn_memtable_table_push(io, state, (const uint8 *) pushes[k].key, 1, &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(1995, 50, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + } + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(1997, 61, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(1997, 9, spilled, "spilled(io, &storage(9743), Of::Rows, mine).unwrap()"); + std_vec_new(&left, sizeof(StdVec), _Alignof(StdVec)); + if (!txn_memtable_table_walked(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, state, txn_memtable_table_tests_collecting, + &left, NULL, &error)) + STD_RESULT_EXPECT_FAILED(2003, 10, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_table_tests_keys_of(left_keys, lengthof(left_keys), &wanted); + STD_ASSERT_EQ(2004, 9, txn_memtable_table_tests_keys_eq(&left, &wanted), &left, + txn_memtable_table_tests_keys_Debug_fmt, &wanted, txn_memtable_table_tests_keys_Debug_fmt); + txn_memtable_table_tests_keys_free(&wanted); + txn_memtable_table_tests_keys_free(&left); + listed = txn_spill_tests_files_of(io, state); + for (size_t k = 0; k < listed.len; k++) + if (STD_VEC_AT(&listed, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_ROWS) + STD_VEC_AT(&listed, TxnSpillFile, kept++) = STD_VEC_AT(&listed, TxnSpillFile, k); + listed.len = kept; + STD_ASSERT_EQ(2007, 9, listed.len == 1, &listed.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt); + file = STD_VEC_AT(&listed, TxnSpillFile, 0); + std_vec_free_Checked(&listed, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + left_tuple = STD_TUPLE(2, STD_ITEM(&file.xid, std_u64_Debug_fmt), STD_ITEM(&file.number, std_u32_Debug_fmt)); + right_tuple = STD_TUPLE(2, STD_ITEM(&mine, std_u64_Debug_fmt), STD_ITEM(&zero, std_u32_Debug_fmt)); + STD_ASSERT_EQ(2009, 9, file.xid == mine && file.number == 0, &left_tuple, std_tuple_Debug_fmt, &right_tuple, + std_tuple_Debug_fmt); + if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, state, &storage, TXN_RUNS_KIND_SPILLED_ROWS, mine, &names, &error)) + STD_RESULT_EXPECT_FAILED(2010, 75, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_runs_spill_directory(9743, txn_state_own(state), &dir); + txn_runs_spill_name(&names, 0, &name); + std_path_join(dir.bytes.ptr, dir.bytes.len, name.ptr, name.len, &path); + std_String_drop(&name); + std_vec_free_Checked(&dir.bytes, 1, 1); + if (!txn_spill_hits(io, &path, &file, (const uint8 *) "a", 1, &hit, &error)) + STD_RESULT_EXPECT_FAILED(2012, 48, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (hit && !txn_spill_hits(io, &path, &file, (const uint8 *) "b", 1, &hit, &error)) + STD_RESULT_EXPECT_FAILED(2012, 91, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2012, 9, hit, + "hits(io, (&path, &file), b\"a\").unwrap() && hits(io, (&path, &file), b\"b\").unwrap()"); + if (!chunks_file_read_file(HOST_IO_IO_READ_DOORS(io), path.bytes.ptr, path.bytes.len, &bytes, &failed)) + STD_RESULT_EXPECT_FAILED(2013, 56, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!txn_runs_spill_hash(bytes.ptr, bytes.len, &hash, &error)) + STD_RESULT_EXPECT_FAILED(2014, 66, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + sized[0] = bytes.len; + sized[1] = hash; + left_tuple = STD_TUPLE(2, STD_ITEM(&sized[0], std_u64_Debug_fmt), STD_ITEM(&sized[1], std_u64_Debug_fmt)); + right_tuple = STD_TUPLE(2, STD_ITEM(&file.len, std_u64_Debug_fmt), STD_ITEM(&file.hash, std_u64_Debug_fmt)); + STD_ASSERT_EQ(2014, 9, sized[0] == file.len && sized[1] == file.hash, &left_tuple, std_tuple_Debug_fmt, + &right_tuple, std_tuple_Debug_fmt); + if (!txn_runs_decode(bytes.ptr, bytes.len, &names, file.hash, &decoded, &error)) + STD_RESULT_EXPECT_FAILED(2015, 63, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_with_capacity_Checked(&read, decoded.keys.len, sizeof(txn_spill_tests_read_t), + _Alignof(txn_spill_tests_read_t)); + for (size_t k = 0; k < decoded.keys.len; k++) + { + const TxnRunsKey *key = &STD_VEC_AT(&decoded.keys, TxnRunsKey, k); + txn_spill_tests_read_t entry = {.key = key->key}; + + std_vec_with_capacity_Checked(&entry.values, key->versions.len, sizeof(StdVec), _Alignof(StdVec)); + for (size_t j = 0; j < key->versions.len; j++) + { + StdVec value = txn_spill_tests_value(&STD_VEC_AT(&key->versions, TxnMemtableVersionRecord, j)); + + std_vec_push_Checked(&entry.values, &value, sizeof(StdVec), _Alignof(StdVec)); + } + std_vec_push_Checked(&read, &entry, sizeof(entry), _Alignof(txn_spill_tests_read_t)); + } + txn_memtable_table_tests_keys_of(a_values, lengthof(a_values), &entries[0].values); + txn_memtable_table_tests_keys_of(b_values, lengthof(b_values), &entries[1].values); + std_slice_to_vec_Checked(&wanted, entries, lengthof(entries), sizeof(txn_spill_tests_read_t), + _Alignof(txn_spill_tests_read_t)); + STD_ASSERT_EQ(2032, 9, txn_spill_tests_reads_eq(&read, &wanted), &read, txn_spill_tests_reads_Debug_fmt, &wanted, + txn_spill_tests_reads_Debug_fmt); + txn_spill_tests_reads_free(&wanted); + txn_spill_tests_reads_free(&read); + txn_runs_Decoded_drop(&decoded); + std_vec_free_Checked(&bytes, 1, 1); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2033, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + chain = txn_spill_tests_chain(io, &hold, state, "b"); + txn_spill_tests_chain_is(2047, &chain, b_xids, b_values, 2); + chain = txn_spill_tests_chain(io, &hold, state, "d"); + txn_spill_tests_chain_is(2048, &chain, d_xids, d_values, 2); + chain = txn_spill_tests_chain(io, &hold, state, "e"); + STD_ASSERT(2049, 9, chain.len == 0, "chain(io, b\"e\").is_empty()"); + std_vec_free_Checked(&chain, sizeof(txn_spill_tests_versioned_t), _Alignof(txn_spill_tests_versioned_t)); + host_delta_Hold_drop(&hold); + std_vec_free_Checked(&path.bytes, 1, 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* txn_spill_tests_Direction_Debug_fmt is Direction's derived Debug, its variant's name, for the tests' asserts. */ +static bool +txn_spill_tests_Direction_Debug_fmt(const void *value, StdFormatter *f) +{ + if (*(const TxnMemtableTableDirection *) value == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD) + return std_fmt_write_str(f, "Forward", 7); + return std_fmt_write_str(f, "Backward", 8); +} + +/* txn_spill_tests_versions_free drops a Vec's versions and frees it. */ +static void +txn_spill_tests_versions_free(StdVec *versions) +{ + for (size_t k = 0; k < versions->len; k++) + txn_memtable_version_Version_drop(&STD_VEC_AT(versions, TxnMemtableVersionVersion, k)); + std_vec_free_Checked(versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); +} + +/* txn_spill_tests_one_version asserts at its places that a number's key has one version in a state's memtable and spills, the message the number when one is asked for. */ +static void +txn_spill_tests_one_version(uint32 line, uint32 col, uint32 unwrap_col, HostIoIo *io, const HostDeltaHold *hold, + HostSharedRecord state, uint64 number, bool numbered) +{ + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[1] = {{&number, std_u64_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + StdVec versions; + HostError error; + size_t one = 1; + uint8 key[8]; + + std_u64_to_be_bytes(number, key); + if (!txn_memtable_table_versions(io, hold, state, key, sizeof(key), &versions, &error)) + STD_RESULT_EXPECT_FAILED(line, unwrap_col, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (versions.len != 1) + std_assert_failed(RUST_FILE, line, col, "==", &versions.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt, + numbered ? &message : NULL); + txn_spill_tests_versions_free(&versions); +} + +/* txn_spill_tests_doubled returns the scan case's key of a number: its double, as big-endian bytes, as its key closure does. */ +static void +txn_spill_tests_doubled(uint32 number, StdVec *result) +{ + uint8 be[4]; + + std_u32_to_be_bytes(number * 2, be); + std_slice_to_vec_Checked(result, be, sizeof(be), 1, 1); +} + +/* + * Bugs row 74's scan: a scan's steps over a storage's rows merge its + * memtable's keys and its spilled keys in key order, both ways, across a + * spill file's chunks, and a key's lookup finds its spilled version in any + * chunk. + */ +void +test_txn_spill_tests_a_scan_merges_spilled_keys_across_chunks_in_order(void) +{ + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + static const TxnMemtableTableDirection ways[] = {TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, + TXN_MEMTABLE_TABLE_DIRECTION_BACKWARD}; + static const uint32 looked[] = {0, 77, 150, 299}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9744); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9744); + uint64 mine = simc_begin().f0; + uint64 other = simc_begin().f0; + uint8 odd[4]; + uint8 three[4]; + uint8 value[1000]; + StdVec expected; + StdVec odd_key; + StdVec versions; + TxnMemtableOp op; + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(mine), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + HostDeltaHold hold; + HostError error; + bool sorted_odd = false; + size_t one = 1; + bool spilled; + + for (uint32 number = 0; number < 300; number++) + { + StdVec key; + + memset(value, (uint8) number, sizeof(value)); + op = txn_memtable_testing_put(value, sizeof(value)); + txn_spill_tests_doubled(number, &key); + if (!txn_memtable_table_push(io, state, key.ptr, key.len, &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(2071, 59, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&key, 1, 1); + txn_memtable_Op_drop(&op); + } + std_u32_to_be_bytes(301, odd); + op = txn_memtable_testing_put((const uint8 *) "o", 1); + change.writer = txn_memtable_testing_writer(other); + if (!txn_memtable_table_push(io, state, odd, sizeof(odd), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(2076, 47, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2077, 61, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2077, 9, spilled, "spilled(io, &storage(9744), Of::Rows, mine).unwrap()"); + std_vec_with_capacity_Checked(&expected, 301, sizeof(StdVec), _Alignof(StdVec)); + for (uint32 number = 0; number < 300; number++) + { + StdVec key; + + txn_spill_tests_doubled(number, &key); + if (!sorted_odd && memcmp(odd, key.ptr, sizeof(odd)) < 0) + { + std_slice_to_vec_Checked(&odd_key, odd, sizeof(odd), 1, 1); + std_vec_push_Checked(&expected, &odd_key, sizeof(StdVec), _Alignof(StdVec)); + sorted_odd = true; + } + std_vec_push_Checked(&expected, &key, sizeof(StdVec), _Alignof(StdVec)); + } + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2081, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t w = 0; w < lengthof(ways); w++) + { + StdVec keys; + StdVec after = {0}; + bool has_after = false; + + std_vec_new(&keys, sizeof(StdVec), _Alignof(StdVec)); + for (;;) + { + txn_TupleVecU8VecVersion_t found; + StdVec copy; + bool some; + + if (!txn_memtable_table_next(io, &hold, state, has_after, after.ptr, after.len, ways[w], &some, &found, + &error)) + STD_RESULT_EXPECT_FAILED(2085, 102, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!some) + break; + STD_ASSERT_EQ(2086, 17, found.f1.len == 1, &found.f1.len, std_usize_Debug_fmt, &one, + std_usize_Debug_fmt); + txn_spill_tests_versions_free(&found.f1); + std_vec_clone(©, &found.f0, 1, 1); + std_vec_push_Checked(&keys, ©, sizeof(StdVec), _Alignof(StdVec)); + if (has_after) + std_vec_free_Checked(&after, 1, 1); + after = found.f0; + has_after = true; + } + if (has_after) + std_vec_free_Checked(&after, 1, 1); + if (ways[w] == TXN_MEMTABLE_TABLE_DIRECTION_BACKWARD) + for (size_t k = 0; k < keys.len / 2; k++) + { + StdVec swap = STD_VEC_AT(&keys, StdVec, k); + + STD_VEC_AT(&keys, StdVec, k) = STD_VEC_AT(&keys, StdVec, keys.len - 1 - k); + STD_VEC_AT(&keys, StdVec, keys.len - 1 - k) = swap; + } + if (!txn_memtable_table_tests_keys_eq(&keys, &expected)) + { + StdFmtArg args[1] = {{&ways[w], txn_spill_tests_Direction_Debug_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_assert_failed(RUST_FILE, 2093, 13, "==", &keys, txn_memtable_table_tests_keys_Debug_fmt, &expected, + txn_memtable_table_tests_keys_Debug_fmt, &message); + } + txn_memtable_table_tests_keys_free(&keys); + } + txn_memtable_table_tests_keys_free(&expected); + for (size_t k = 0; k < lengthof(looked); k++) + { + StdVec key; + TxnMemtableVersionRecord record; + uint64 xid; + + txn_spill_tests_doubled(looked[k], &key); + if (!txn_memtable_table_versions(io, &hold, state, key.ptr, key.len, &versions, &error)) + STD_RESULT_EXPECT_FAILED(2096, 76, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&key, 1, 1); + STD_ASSERT_EQ(2097, 13, versions.len == 1, &versions.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt); + record = txn_memtable_version_Version_record(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, 0)); + xid = txn_memtable_version_Record_writer(&record).xid; + STD_ASSERT_EQ(2098, 13, xid == mine, &xid, std_u64_Debug_fmt, &mine, std_u64_Debug_fmt); + txn_spill_tests_versions_free(&versions); + } + std_u32_to_be_bytes(3, three); + if (!txn_memtable_table_versions(io, &hold, state, three, sizeof(three), &versions, &error)) + STD_RESULT_EXPECT_FAILED(2100, 72, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2100, 9, versions.len == 0, "table::versions(io, &hold, state, &3u32.to_be_bytes()).unwrap().is_empty()"); + txn_spill_tests_versions_free(&versions); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +#include "doltgres/txn/values.h" + +/* txn_spill_tests_newest_is asserts at its line that a value id's newest entry, in a state's value map or in its spilled value maps, is a wanted one or nothing. */ +static void +txn_spill_tests_newest_is(uint32 line, uint32 unwrap_col, HostIoIo *io, HostSharedRecord state, bool spilled, + uint32 id, bool has_expected, const TxnMemtableMapEntry *expected) +{ + TxnMemtableMapEntry entry; + HostError error; + bool some; + bool ok; + StdOptionRef left; + StdOptionRef right = {has_expected, STD_ITEM(expected, txn_memtable_map_tests_seen_Debug_fmt)}; + + if (spilled) + ok = txn_spill_newest_value(io, state, id, &some, &entry, &error); + else + ok = txn_memtable_map_newest(io, state, TXN_MEMTABLE_MAP_MAP_VALUES, id, &some, &entry, &error); + if (!ok) + STD_RESULT_EXPECT_FAILED(line, unwrap_col, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + left = (StdOptionRef) + { + some, STD_ITEM(&entry, txn_memtable_map_tests_seen_Debug_fmt) + }; + STD_ASSERT_EQ(line, 9, some == has_expected && (!some || txn_memtable_map_tests_seen_eq(&entry, expected)), + &left, std_option_Debug_fmt, &right, std_option_Debug_fmt); + if (some) + txn_memtable_map_tests_seen_drop(&entry); +} + +/* txn_spill_tests_mark_Debug_fmt is a value map's mark's Debug, the tuple (u64, u64). */ +static bool +txn_spill_tests_mark_Debug_fmt(const void *value, StdFormatter *f) +{ + const std_TupleU64U64_t *mark = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&mark->f0, std_u64_Debug_fmt), STD_ITEM(&mark->f1, std_u64_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* + * Bugs row 74's value map: a spill of a storage's value map takes out the + * entries only its transaction wrote, and a reader finds a spilled entry's + * newest below the memtable, a removal included, and moves the state's mark. + */ +void +test_txn_spill_tests_a_spilled_value_maps_entries_are_read_below_the_memtable(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9745); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9745); + uint64 mine = simc_begin().f0; + uint64 other = simc_begin().f0; + const struct + { + uint64 xid; + uint32 id; + const char *value; + } pushes[] = {{mine, 7, "e"}, {mine, 8, "f"}, {mine, 8, NULL}, {other, 9, "g"}}; + TxnMemtableMapEntry e = {.has_value = true}; + TxnMemtableMapEntry removal = {.has_value = false}; + TxnMemtableMapEntry g = {.has_value = true}; + std_TupleU64U64_t before; + std_TupleU64U64_t after; + HostError error; + bool spilled; + + for (size_t k = 0; k < lengthof(pushes); k++) + { + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_DELETE}; + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + uint8 key[4]; + + if (pushes[k].value != NULL) + op = txn_memtable_testing_put((const uint8 *) pushes[k].value, 1); + std_u32_to_be_bytes(pushes[k].id, key); + if (!txn_memtable_table_alone_State(io, state, true, TXN_MEMTABLE_MAP_MAP_VALUES, key, sizeof(key), &change, + &handle, &error)) + STD_RESULT_EXPECT_FAILED(2122, 84, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + } + before = txn_values_mark(state); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_VALUES, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2125, 63, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2125, 9, spilled, "spilled(io, &storage(9745), Of::Values, mine).unwrap()"); + after = txn_values_mark(state); + if (after.f0 == before.f0 && after.f1 == before.f1) + std_assert_failed(RUST_FILE, 2126, 9, "!=", &after, txn_spill_tests_mark_Debug_fmt, &before, + txn_spill_tests_mark_Debug_fmt, NULL); + std_slice_to_vec_Checked(&e.value, (const uint8 *) "e", 1, 1, 1); + std_slice_to_vec_Checked(&g.value, (const uint8 *) "g", 1, 1, 1); + txn_spill_tests_newest_is(2127, 59, io, state, false, 7, false, NULL); + txn_spill_tests_newest_is(2128, 47, io, state, true, 7, true, &e); + txn_spill_tests_newest_is(2129, 47, io, state, true, 8, true, &removal); + txn_spill_tests_newest_is(2130, 47, io, state, true, 9, false, NULL); + txn_spill_tests_newest_is(2131, 59, io, state, false, 9, true, &g); + std_vec_free_Checked(&g.value, 1, 1); + std_vec_free_Checked(&e.value, 1, 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +#define TXN_MEMTABLE_TABLE_RS Unit +#define TXN_MEMTABLE_TABLE_RETURNS bool +#define TXN_MEMTABLE_TABLE_RESULT_PARAMS , HostError *error +#define TXN_MEMTABLE_TABLE_DECLARE +#include "doltgres/txn/memtable/table_template.h" + +/* txn_spill_tests_kinds_Debug_fmt is a Vec's Debug. */ +static bool +txn_spill_tests_kinds_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *kinds = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < kinds->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(kinds, TxnRunsKind, k), txn_spill_tests_Kind_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_spill_tests_kinds_are asserts at its place that listed files are of a list of kinds, newest first. */ +static void +txn_spill_tests_kinds_are(uint32 line, const StdVec *files, const TxnRunsKind *wanted, size_t wanted_len) +{ + StdVec kinds; + StdVec wanted_vec = {(void *) wanted, wanted_len, wanted_len}; + bool eq = files->len == wanted_len; + + std_vec_with_capacity_Checked(&kinds, files->len, sizeof(TxnRunsKind), _Alignof(TxnRunsKind)); + for (size_t k = 0; k < files->len; k++) + std_vec_push_Checked(&kinds, &STD_VEC_AT(files, TxnSpillFile, k).kind, sizeof(TxnRunsKind), + _Alignof(TxnRunsKind)); + for (size_t k = 0; eq && k < wanted_len; k++) + eq = STD_VEC_AT(&kinds, TxnRunsKind, k) == wanted[k]; + STD_ASSERT_EQ(line, 9, eq, &kinds, txn_spill_tests_kinds_Debug_fmt, &wanted_vec, txn_spill_tests_kinds_Debug_fmt); + std_vec_free_Checked(&kinds, sizeof(TxnRunsKind), _Alignof(TxnRunsKind)); +} + +/* txn_spill_tests_tuple_is asserts at its line that a tid's row's key in a state's tuple id table, or in its spilled tuple ids, is a wanted key or nothing. */ +static void +txn_spill_tests_tuple_is(uint32 line, uint32 col, uint32 unwrap_col, HostIoIo *io, HostSharedRecord state, uint64 tid, + const TxnMemtableMapEntry *expected) +{ + HostRow row = {tid}; + TxnMemtableMapEntry found; + HostError error; + + if (!txn_memtable_tuples_get(io, state, row, &found.has_value, &found.value, &error)) + STD_RESULT_EXPECT_FAILED(line, unwrap_col, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ(line, col, txn_memtable_map_tests_seen_eq(&found, expected), &found, + txn_memtable_map_tests_seen_Debug_fmt, expected, txn_memtable_map_tests_seen_Debug_fmt); + txn_memtable_map_tests_seen_drop(&found); +} + +/* + * Bugs row 74's tuple ids: a spill of a storage's rows takes its rows' tuple + * ids out of the tuple id table into a file of spilled tuple ids, and each + * tid still finds its row's key, the other writer's from the table. + */ +void +test_txn_spill_tests_a_spills_tuple_ids_still_find_their_rows_keys(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9746); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9746); + uint64 mine = simc_begin().f0; + uint64 other = simc_begin().f0; + const struct + { + uint64 xid; + uint64 number; + } writes[] = {{mine, 1}, {mine, 2}, {mine, 3}, {other, 4}}; + static const TxnRunsKind wanted[] = {TXN_RUNS_KIND_SPILLED_TUPLES, TXN_RUNS_KIND_SPILLED_ROWS}; + TxnMemtableMapEntry none = {.has_value = false}; + StdVec files; + HostError error; + bool spilled; + + for (size_t k = 0; k < lengthof(writes); k++) + { + uint64 tid = UINT64CONST(1) << 16 | writes[k].number; + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(writes[k].xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + txn_memtable_tuples_tests_stored_t captures = {.tid = tid, .change = &change}; + txn_TupleRefSliceU8Usize_t rows[1]; + uint8 key[8]; + + op.u.put.tuple.f0 = tid; + std_u64_to_be_bytes(writes[k].number, key); + rows[0].f0 = (StdBytes) + { + key, sizeof(key) + }; + rows[0].f1 = txn_memtable_version_encoded(&change.writer, change.op, change.shown, false, 0); + if (!txn_memtable_table_prepare_Unit(io, state, rows, lengthof(rows), NULL, 0, true, + txn_memtable_tuples_tests_stored_inside, &captures, NULL, &error)) + STD_RESULT_EXPECT_FAILED(2161, 14, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + } + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2163, 61, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2163, 9, spilled, "spilled(io, &storage(9746), Of::Rows, mine).unwrap()"); + files = txn_spill_tests_files_of(io, state); + txn_spill_tests_kinds_are(2165, &files, wanted, lengthof(wanted)); + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + for (uint64 number = 1; number <= 4; number++) + { + TxnMemtableMapEntry key = {.has_value = true}; + uint8 be[8]; + + std_u64_to_be_bytes(number, be); + std_slice_to_vec_Checked(&key.value, be, sizeof(be), 1, 1); + txn_spill_tests_tuple_is(2167, 13, 65, io, state, UINT64CONST(1) << 16 | number, &key); + std_vec_free_Checked(&key.value, 1, 1); + } + txn_spill_tests_tuple_is(2169, 9, 56, io, state, UINT64CONST(1) << 16 | 9, &none); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +#include "doltgres/txn/fold.h" +#include "doltgres/txn/insert.h" + +/* TXN_SPILL_TESTS_WRITERS are the three writers' names and databases. */ +static const struct +{ + const char *name; + uint32 database; +} TXN_SPILL_TESTS_WRITERS[] = {{"w1", 9484}, {"w2", 9485}, {"w3", 9486}}; + +/* txn_spill_tests_tree returns the three writers' case's storage of a tree of a database, as its trees closure numbers them. */ +static HostStorage +txn_spill_tests_tree(uint32 database, uint32 tree) +{ + HostStorage storage = txn_memtable_testing_storage(database); + + storage.relation = 16384 + tree; + return storage; +} + +/* txn_spill_tests_based_t is the three writers' case's base of a tree: its repository, a shared read, and its state. */ +typedef struct txn_spill_tests_based_t +{ + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; +} txn_spill_tests_based_t; + +/* txn_spill_tests_based opens a tree's repository and a shared read, and makes the tree's state, keyless and permanent, as the three writers' case's based closure does. */ +static bool +txn_spill_tests_based(HostIoIo *io, const HostStorage *tree, txn_spill_tests_based_t *result, HostError *error) +{ + TxnStateMade made = {.key = {.tag = TXN_BASE_KEY_NONE}, .table = tree->relation}; + ChunksErrorError failed; + + if (!repo_Repo_open(io, tree->database, HOST_PG_DEFAULT, &result->repo, &failed)) + { + *error = txn_base_storage(&failed); + return false; + } + if (!host_shared_read(io, &result->reading, error)) + { + repo_Repo_drop(&result->repo); + return false; + } + if (!txn_state_state(io, &result->repo, &result->reading, tree, TXN_STATE_PERSISTENCE_PERMANENT, true, &made, + &result->held, error)) + { + host_shared_Reader_drop(&result->reading); + repo_Repo_drop(&result->repo); + return false; + } + return true; +} + +/* txn_spill_tests_based_drop drops a tree's base: its state, its read, then its repository. */ +static void +txn_spill_tests_based_drop(txn_spill_tests_based_t *based) +{ + host_shared_Held_drop(&based->held); + host_shared_Reader_drop(&based->reading); + repo_Repo_drop(&based->repo); +} + +/* txn_spill_tests_three_t is the three writers' case's shared state: its trees and rows each, whether it folds, its counts of writers finished and ended, the databases committed, and whether a writer failed. */ +typedef struct txn_spill_tests_three_t +{ + uint32 trees_each; + uint64 rows; + bool folding; + size_t finished; + size_t ended; + /* committed is a Vec. */ + StdVec committed; + bool failed; +} txn_spill_tests_three_t; + +/* txn_spill_tests_writing_t is a writer's captures in the three writers' case: the shared state and its database. */ +typedef struct txn_spill_tests_writing_t +{ + txn_spill_tests_three_t *three; + uint32 database; +} txn_spill_tests_writing_t; + +/* txn_spill_tests_states_free drops a writer's trees' states and frees them. */ +static void +txn_spill_tests_states_free(StdVec *states) +{ + for (size_t k = 0; k < states->len; k++) + host_shared_Held_drop(&STD_VEC_AT(states, HostSharedHeld, k)); + std_vec_free_Checked(states, sizeof(HostSharedHeld), _Alignof(HostSharedHeld)); +} + +/* txn_spill_tests_pushed writes each row of a writer's trees, a key at a time across its trees, each counted in the transaction's written trees first. */ +static bool +txn_spill_tests_pushed(const txn_spill_tests_writing_t *c, HostIoIo *io, const StdVec *states, HostXid xid, + const TxnMemtableOp *op, HostError *error) +{ + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid.f0), .op = op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + size_t need = txn_memtable_version_encoded(&change.writer, change.op, change.shown, change.has_lsn, + change.lsn); + + for (uint64 number = 0; number < c->three->rows; number++) { - TxnMemtableVersionVersion *version = &STD_VEC_AT(&versions, TxnMemtableVersionVersion, k); - TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); - txn_spill_tests_versioned_t entry = {.xid = txn_memtable_version_Record_writer(&record).xid, - .value = txn_spill_tests_value(&record)}; + uint8 key[8]; - std_vec_push_Checked(&chain, &entry, sizeof(entry), _Alignof(txn_spill_tests_versioned_t)); - txn_memtable_version_Version_drop(version); + std_u64_to_be_bytes(number, key); + for (size_t k = 0; k < states->len; k++) + { + HostStorage tree = txn_spill_tests_tree(c->database, (uint32) k); + TxnMemtableVersionHandle handle; + uint64 touched; + + if (!txn_written_touched(io, xid.f0, &tree, TXN_WRITTEN_OF_ROWS, key, sizeof(key), need, &touched, + error) || + !txn_memtable_table_push(io, host_shared_Held_record(&STD_VEC_AT(states, HostSharedHeld, k)), key, + sizeof(key), &change, &handle, error)) + return false; + } } - std_vec_free_Checked(&versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); - return chain; + return true; +} + +/* txn_spill_tests_wrote is a writer's body in the three writers' case, its wrote closure: it writes its trees' rows, waits for the others or commits, and ends its writes. */ +static bool +txn_spill_tests_wrote(const txn_spill_tests_writing_t *c, HostIoIo *io, HostError *error) +{ + txn_spill_tests_three_t *three = c->three; + uint8 value[1000]; + StdVec states; + HostXid xid; + TxnMemtableOp op; + bool ok; + + std_vec_new(&states, sizeof(HostSharedHeld), _Alignof(HostSharedHeld)); + for (uint32 tree = 0; tree < three->trees_each; tree++) + { + HostStorage storage = txn_spill_tests_tree(c->database, tree); + HostSharedHeld held; + + if (!txn_memtable_testing_held(io, &storage, &held, error)) + { + txn_spill_tests_states_free(&states); + return false; + } + std_vec_push_Checked(&states, &held, sizeof(held), _Alignof(HostSharedHeld)); + } + xid = simc_begin(); + memset(value, 7, sizeof(value)); + op = txn_memtable_testing_put(value, sizeof(value)); + ok = txn_spill_tests_pushed(c, io, &states, xid, &op, error); + txn_memtable_Op_drop(&op); + txn_spill_tests_states_free(&states); + if (!ok) + return false; + three->finished++; + while (!three->folding && three->finished < lengthof(TXN_SPILL_TESTS_WRITERS)) + { + if ((size_t) simc_delta_waiting() + three->finished >= lengthof(TXN_SPILL_TESTS_WRITERS)) + { + *error = txn_spill_internal("every writer still writing sleeps at the budget"); + return false; + } + if (!host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-written", error)) + return false; + } + if (three->folding) + { + TxnStatusOutcome committed = {.tag = TXN_STATUS_OUTCOME_COMMITTED}; + TxnStatusRoomed roomed; + bool took; + + if (!host_wal_insert(HOST_IO_IO_IO_DOORS(io), HOST_WAL_KIND_ROW_CHANGE, NULL, 0, &committed.u.committed, + error) || + !txn_status_take(io, xid, committed, &roomed, &took, error)) + return false; + simc_commit(xid); + std_vec_push_Checked(&three->committed, &c->database, sizeof(uint32), _Alignof(uint32)); + } + if (!txn_written_ended(io, error)) + return false; + three->ended++; + return true; +} + +/* txn_spill_tests_writer is a writer's actor in the three writers' case: its body, a failure marked in the shared state. */ +static bool +txn_spill_tests_writer(void *captures, HostIoIo *io, HostError *error) +{ + const txn_spill_tests_writing_t *c = captures; + bool wrote = txn_spill_tests_wrote(c, io, error); + + c->three->failed |= !wrote; + return wrote; +} + +/* txn_spill_tests_folded folds each tree of each database committed, as the three writers' case's folder does when a writer waits. */ +static bool +txn_spill_tests_folded(const txn_spill_tests_three_t *three, HostIoIo *io, const StdVec *all, HostError *error) +{ + const TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + const TxnInsertPage page = {.bytes = 8168, .most = 291}; + + for (size_t k = 0; k < all->len; k++) + for (uint32 tree = 0; tree < three->trees_each; tree++) + { + HostStorage storage = txn_spill_tests_tree(STD_VEC_AT(all, uint32, k), tree); + txn_spill_tests_based_t based; + StdVec warned; + HostLsn point; + bool has_point; + bool folded; + bool ok; + + if (!txn_spill_tests_based(io, &storage, &based, error)) + return false; + std_vec_new(&warned, sizeof(StdString), _Alignof(StdString)); + ok = txn_fold_point(io, &has_point, &point, error) && + txn_fold_fold_now(io, &based.repo, &based.reading, &based.held, &order, false, has_point, point, + PG_UINT64_MAX, &warned, &page, NULL, &folded, error); + std_vec_drop_Checked(&warned, sizeof(StdString), _Alignof(StdString), std_String_drop); + txn_spill_tests_based_drop(&based); + if (!ok) + return false; + } + return true; +} + +/* txn_spill_tests_folder is the three writers' case's folder: while a writer is unended, it folds the committed databases' trees whenever a writer waits at the budget and a database committed since, or every five thousand idle steps. */ +static bool +txn_spill_tests_folder(void *captures, HostIoIo *io, HostError *error) +{ + txn_spill_tests_three_t *three = captures; + size_t folded = 0; + int32 idle = 0; + + while (three->ended < lengthof(TXN_SPILL_TESTS_WRITERS)) + { + StdVec all; + bool fresh; + bool ok = true; + + if (three->failed) + { + *error = txn_spill_internal("a writer failed, so the fold has nothing to wait for"); + return false; + } + std_vec_clone(&all, &three->committed, sizeof(uint32), _Alignof(uint32)); + fresh = all.len > folded; + idle = !fresh && simc_delta_waiting() > 0 ? idle + 1 : 0; + if (idle > 100000) + { + std_vec_free_Checked(&all, sizeof(uint32), _Alignof(uint32)); + *error = txn_spill_internal("every writer still writing sleeps with nothing committed to fold"); + return false; + } + if (simc_delta_waiting() > 0 && (fresh || idle % 5000 == 4999)) + { + folded = all.len; + ok = txn_spill_tests_folded(three, io, &all, error); + } + std_vec_free_Checked(&all, sizeof(uint32), _Alignof(uint32)); + if (!ok || !host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-folding", error)) + return false; + } + return true; } /* - * Bugs row 74's spill: a spill takes out of the memtable exactly the keys - * whose chain holds only its transaction's versions, writes them, each key's - * versions newest first, to a synced file its entry names, and leaves a key - * another writer's version is in, whatever the order of the two. + * txn_spill_tests_three_writers runs bugs row 124's three writers, each of a + * count of trees of a count of rows: holding their commits until all three + * have written, or, folding, each committing once written while a fold runs, + * its commit at an empty record's end, since the simulation writes no commit + * record, so the fold's point passes it. */ -void -test_txn_spill_tests_a_spill_takes_only_the_keys_its_transaction_alone_holds(void) +static void +txn_spill_tests_three_writers(uint32 trees_each, uint64 rows, bool folding) { - static const char *const left_keys[] = {"c", "d"}; - static const char *const a_values[] = {"1"}; - static const char *const b_values[] = {"3", "2"}; - static const char *const d_values[] = {"6", "5"}; - SimcTestIo test_io = simc_io(); - HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9743); - HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9743); - uint64 mine = simc_begin().f0; - uint64 other = simc_begin().f0; - const struct - { - uint64 xid; - const char *key; - const char *value; - } pushes[] = {{mine, "a", "1"}, {mine, "b", "2"}, {mine, "b", "3"}, {other, "c", "4"}, {other, "d", "5"}, - {mine, "d", "6"}}; - uint64 b_xids[2] = {mine, mine}; - uint64 d_xids[2] = {mine, other}; - uint32 zero = 0; - uint64 sized[2]; - txn_spill_tests_read_t entries[2] = {{.key = {(const uint8 *) "a", 1}}, {.key = {(const uint8 *) "b", 1}}}; - StdTupleRef left_tuple; - StdTupleRef right_tuple; - StdVec left; - StdVec wanted; - StdVec listed; - StdVec bytes; - StdVec read; - StdVec chain; - StdPathBuf dir; - StdPathBuf path; - StdString name; - TxnRunsNames names; - TxnRunsDecoded decoded; - TxnSpillFile file; + static const StdFmtPiece unfinished[] = {STD_FMT_LIT("the writers didn't finish, "), STD_FMT_ARG(0), + STD_FMT_LIT(" row and "), STD_FMT_ARG(1), STD_FMT_LIT(" tuple-id entries in "), STD_FMT_ARG(2), + STD_FMT_LIT(" bytes: "), STD_FMT_ARG(3)}; + static const StdFmtPiece outcome[] = {STD_FMT_ARG(0)}; + txn_spill_tests_three_t three = {.trees_each = trees_each, .rows = rows, .folding = folding}; + txn_spill_tests_writing_t writing[lengthof(TXN_SPILL_TESTS_WRITERS)]; + SimcEnding returned = {.tag = SIMC_ENDING_RETURNED}; + SimcTestIo test_io; + HostIoIo *io; + SimcRun run; + SimcReport report; HostDeltaHold hold; - ChunksErrorError failed; HostError error; - size_t kept = 0; - size_t one = 1; - uint64 hash; - bool spilled; - bool hit; + bool all; - for (size_t k = 0; k < lengthof(pushes); k++) + simc_size_delta(2 << 20); + test_io = simc_io(); + io = &test_io.io; + for (size_t w = 0; w < lengthof(TXN_SPILL_TESTS_WRITERS); w++) { - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) pushes[k].value, 1); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - TxnMemtableVersionHandle handle; + RepoRepo repo; + ChunksErrorError failed; - if (!txn_memtable_table_push(io, state, (const uint8 *) pushes[k].key, 1, &change, &handle, &error)) - STD_RESULT_EXPECT_FAILED(1995, 50, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_memtable_Op_drop(&op); + if (!repo_Repo_create(io, TXN_SPILL_TESTS_WRITERS[w].database, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(2215, 84, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + repo_Repo_drop(&repo); + for (uint32 tree = 0; tree < trees_each; tree++) + { + HostStorage storage = txn_spill_tests_tree(TXN_SPILL_TESTS_WRITERS[w].database, tree); + txn_spill_tests_based_t based; + + if (!txn_spill_tests_based(io, &storage, &based, &error)) + STD_RESULT_EXPECT_FAILED(2217, 39, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_spill_tests_based_drop(&based); + } } - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(1997, 61, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(1997, 9, spilled, "spilled(io, &storage(9743), Of::Rows, mine).unwrap()"); - std_vec_new(&left, sizeof(StdVec), _Alignof(StdVec)); - if (!txn_memtable_table_walked(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, state, txn_memtable_table_tests_collecting, - &left, NULL, &error)) - STD_RESULT_EXPECT_FAILED(2003, 10, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_memtable_table_tests_keys_of(left_keys, lengthof(left_keys), &wanted); - STD_ASSERT_EQ(2004, 9, txn_memtable_table_tests_keys_eq(&left, &wanted), &left, - txn_memtable_table_tests_keys_Debug_fmt, &wanted, txn_memtable_table_tests_keys_Debug_fmt); - txn_memtable_table_tests_keys_free(&wanted); - txn_memtable_table_tests_keys_free(&left); - listed = txn_spill_tests_files_of(io, state); - for (size_t k = 0; k < listed.len; k++) - if (STD_VEC_AT(&listed, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_ROWS) - STD_VEC_AT(&listed, TxnSpillFile, kept++) = STD_VEC_AT(&listed, TxnSpillFile, k); - listed.len = kept; - STD_ASSERT_EQ(2007, 9, listed.len == 1, &listed.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt); - file = STD_VEC_AT(&listed, TxnSpillFile, 0); - std_vec_free_Checked(&listed, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - left_tuple = STD_TUPLE(2, STD_ITEM(&file.xid, std_u64_Debug_fmt), STD_ITEM(&file.number, std_u32_Debug_fmt)); - right_tuple = STD_TUPLE(2, STD_ITEM(&mine, std_u64_Debug_fmt), STD_ITEM(&zero, std_u32_Debug_fmt)); - STD_ASSERT_EQ(2009, 9, file.xid == mine && file.number == 0, &left_tuple, std_tuple_Debug_fmt, &right_tuple, - std_tuple_Debug_fmt); - if (!txn_spill_names(&TXN_SPILL_STATE_HOME_OPS, state, &storage, TXN_RUNS_KIND_SPILLED_ROWS, mine, &names, &error)) - STD_RESULT_EXPECT_FAILED(2010, 75, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_runs_spill_directory(9743, txn_state_own(state), &dir); - txn_runs_spill_name(&names, 0, &name); - std_path_join(dir.bytes.ptr, dir.bytes.len, name.ptr, name.len, &path); - std_String_drop(&name); - std_vec_free_Checked(&dir.bytes, 1, 1); - if (!txn_spill_hits(io, &path, &file, (const uint8 *) "a", 1, &hit, &error)) - STD_RESULT_EXPECT_FAILED(2012, 48, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (hit && !txn_spill_hits(io, &path, &file, (const uint8 *) "b", 1, &hit, &error)) - STD_RESULT_EXPECT_FAILED(2012, 91, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2012, 9, hit, - "hits(io, (&path, &file), b\"a\").unwrap() && hits(io, (&path, &file), b\"b\").unwrap()"); - if (!chunks_file_read_file(HOST_IO_IO_READ_DOORS(io), path.bytes.ptr, path.bytes.len, &bytes, &failed)) - STD_RESULT_EXPECT_FAILED(2013, 56, "called `Result::unwrap()` on an `Err` value", &failed, - chunks_error_Error_Debug_fmt); - if (!txn_runs_spill_hash(bytes.ptr, bytes.len, &hash, &error)) - STD_RESULT_EXPECT_FAILED(2014, 66, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - sized[0] = bytes.len; - sized[1] = hash; - left_tuple = STD_TUPLE(2, STD_ITEM(&sized[0], std_u64_Debug_fmt), STD_ITEM(&sized[1], std_u64_Debug_fmt)); - right_tuple = STD_TUPLE(2, STD_ITEM(&file.len, std_u64_Debug_fmt), STD_ITEM(&file.hash, std_u64_Debug_fmt)); - STD_ASSERT_EQ(2014, 9, sized[0] == file.len && sized[1] == file.hash, &left_tuple, std_tuple_Debug_fmt, - &right_tuple, std_tuple_Debug_fmt); - if (!txn_runs_decode(bytes.ptr, bytes.len, &names, file.hash, &decoded, &error)) - STD_RESULT_EXPECT_FAILED(2015, 63, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - std_vec_with_capacity_Checked(&read, decoded.keys.len, sizeof(txn_spill_tests_read_t), - _Alignof(txn_spill_tests_read_t)); - for (size_t k = 0; k < decoded.keys.len; k++) + std_vec_new(&three.committed, sizeof(uint32), _Alignof(uint32)); + run = simc_Run_new(1); + for (size_t w = 0; w < lengthof(TXN_SPILL_TESTS_WRITERS); w++) { - const TxnRunsKey *key = &STD_VEC_AT(&decoded.keys, TxnRunsKey, k); - txn_spill_tests_read_t entry = {.key = key->key}; + SimcActor writer = {txn_spill_tests_writer, &writing[w]}; - std_vec_with_capacity_Checked(&entry.values, key->versions.len, sizeof(StdVec), _Alignof(StdVec)); - for (size_t j = 0; j < key->versions.len; j++) + writing[w] = (txn_spill_tests_writing_t) { - StdVec value = txn_spill_tests_value(&STD_VEC_AT(&key->versions, TxnMemtableVersionRecord, j)); + &three, TXN_SPILL_TESTS_WRITERS[w].database + }; + simc_Run_actor(&run, TXN_SPILL_TESTS_WRITERS[w].name, strlen(TXN_SPILL_TESTS_WRITERS[w].name), writer); + } + if (folding) + { + SimcActor folder = {txn_spill_tests_folder, &three}; - std_vec_push_Checked(&entry.values, &value, sizeof(StdVec), _Alignof(StdVec)); + simc_Run_actor(&run, "folder", 6, folder); + } + report = simc_Run_finish(run); + if (report.outcome.tag != SIMC_OUTCOME_ENDED) + { + size_t entries = 0; + size_t tuples = 0; + uint64 used; + + for (size_t w = 0; w < lengthof(TXN_SPILL_TESTS_WRITERS); w++) + for (uint32 tree = 0; tree < trees_each; tree++) + { + HostStorage storage = txn_spill_tests_tree(TXN_SPILL_TESTS_WRITERS[w].database, tree); + HostSharedHeld state; + StdVec files; + + if (!txn_memtable_testing_held(io, &storage, &state, &error)) + STD_RESULT_EXPECT_FAILED(2317, 49, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + files = txn_spill_tests_files_of(io, host_shared_Held_record(&state)); + for (size_t k = 0; k < files.len; k++) + { + if (STD_VEC_AT(&files, TxnSpillFile, k).kind != TXN_RUNS_KIND_SPILLED_TUPLES) + entries++; + if (STD_VEC_AT(&files, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_TUPLES) + tuples++; + } + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + host_shared_Held_drop(&state); + } + used = host_delta_used(); + { + StdFmtArg args[4] = {{&entries, std_usize_Display_fmt}, {&tuples, std_usize_Display_fmt}, + {&used, std_u64_Display_fmt}, {&report.outcome, simc_Outcome_Debug_fmt}}; + StdFmtArguments message = {unfinished, lengthof(unfinished), args}; + + std_panic_fmt(RUST_FILE, 2323, 13, &message); } - std_vec_push_Checked(&read, &entry, sizeof(entry), _Alignof(txn_spill_tests_read_t)); } - txn_memtable_table_tests_keys_of(a_values, lengthof(a_values), &entries[0].values); - txn_memtable_table_tests_keys_of(b_values, lengthof(b_values), &entries[1].values); - std_slice_to_vec_Checked(&wanted, entries, lengthof(entries), sizeof(txn_spill_tests_read_t), - _Alignof(txn_spill_tests_read_t)); - STD_ASSERT_EQ(2032, 9, txn_spill_tests_reads_eq(&read, &wanted), &read, txn_spill_tests_reads_Debug_fmt, &wanted, - txn_spill_tests_reads_Debug_fmt); - txn_spill_tests_reads_free(&wanted); - txn_spill_tests_reads_free(&read); - txn_runs_Decoded_drop(&decoded); - std_vec_free_Checked(&bytes, 1, 1); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2033, 36, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - chain = txn_spill_tests_chain(io, &hold, state, "b"); - txn_spill_tests_chain_is(2047, &chain, b_xids, b_values, 2); - chain = txn_spill_tests_chain(io, &hold, state, "d"); - txn_spill_tests_chain_is(2048, &chain, d_xids, d_values, 2); - chain = txn_spill_tests_chain(io, &hold, state, "e"); - STD_ASSERT(2049, 9, chain.len == 0, "chain(io, b\"e\").is_empty()"); - std_vec_free_Checked(&chain, sizeof(txn_spill_tests_versioned_t), _Alignof(txn_spill_tests_versioned_t)); - host_delta_Hold_drop(&hold); - std_vec_free_Checked(&path.bytes, 1, 1); - host_shared_Held_drop(&held); + all = true; + for (size_t k = 0; all && k < report.outcome.u.ended.len; k++) + all = simc_Ending_PartialEq_eq(&STD_VEC_AT(&report.outcome.u.ended, simc_TupleStringEnding_t, k).f1, + &returned); + if (!all) + { + StdFmtArg args[1] = {{&report.outcome, simc_Outcome_Debug_fmt}}; + StdFmtArguments message = {outcome, lengthof(outcome), args}; + + std_panic_fmt(RUST_FILE, 2329, 9, &message); + } + if (!folding) + { + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2333, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t w = 0; w < lengthof(TXN_SPILL_TESTS_WRITERS); w++) + for (uint32 tree = 0; tree < trees_each; tree++) + { + HostStorage storage = txn_spill_tests_tree(TXN_SPILL_TESTS_WRITERS[w].database, tree); + HostSharedHeld state; + uint64 numbers[2] = {0, rows - 1}; + + if (!txn_memtable_testing_held(io, &storage, &state, &error)) + STD_RESULT_EXPECT_FAILED(2336, 45, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; k < lengthof(numbers); k++) + txn_spill_tests_one_version(2338, 21, 98, io, &hold, host_shared_Held_record(&state), numbers[k], + false); + host_shared_Held_drop(&state); + } + host_delta_Hold_drop(&hold); + } + std_vec_free_Checked(&three.committed, sizeof(uint32), _Alignof(uint32)); + simc_Report_drop(&report); simc_TestIo_drop(&test_io); } -/* txn_spill_tests_Direction_Debug_fmt is Direction's derived Debug, its variant's name, for the tests' asserts. */ -static bool -txn_spill_tests_Direction_Debug_fmt(const void *value, StdFormatter *f) +/* + * Bugs row 124's three writers: three transactions each write many small + * trees a row at a time, interleaved at every yield point by the run's seed, + * so their trees' small pages share the delta's blocks, and each writes past + * a third of the budget. A writer refused at it spills its trees one after + * another, and each part's removal frees pages whose blocks the others' pages + * keep, so no block frees; each part's entry takes the slot its own removal + * claimed. No writer commits until all three have written, so every version + * is a waiter's to spill, and every writer finishes with its rows read back. + */ +void +test_txn_spill_tests_three_writers_of_small_trees_past_a_third_of_the_budget_each_spill_and_finish(void) { - if (*(const TxnMemtableTableDirection *) value == TXN_MEMTABLE_TABLE_DIRECTION_FORWARD) - return std_fmt_write_str(f, "Forward", 7); - return std_fmt_write_str(f, "Backward", 8); + txn_spill_tests_three_writers(16, 40, false); } -/* txn_spill_tests_versions_free drops a Vec's versions and frees it. */ -static void -txn_spill_tests_versions_free(StdVec *versions) +/* + * Bugs row 124's committed load: the three writers of twice the trees of half + * the rows each commit once written, so the budget the last one waits at is + * held by committed versions no waiter can spill; a fold of the committed + * storages, which an actor runs while a writer waits, as the server's fold + * trigger does, again while the writer still waits, since a fold refused by + * an older snapshot installs later, frees them, and every writer finishes. + */ +void +test_txn_spill_tests_three_writers_waiting_on_committed_versions_finish_once_a_fold_installs(void) { - for (size_t k = 0; k < versions->len; k++) - txn_memtable_version_Version_drop(&STD_VEC_AT(versions, TxnMemtableVersionVersion, k)); - std_vec_free_Checked(versions, sizeof(TxnMemtableVersionVersion), _Alignof(TxnMemtableVersionVersion)); + txn_spill_tests_three_writers(32, 20, true); } -/* txn_spill_tests_one_version asserts at its places that a number's key has one version in a state's memtable and spills, the message the number when one is asked for. */ +/* txn_spill_tests_returned asserts at its lines that a run's actors all ended, panicking with its lead-in and the outcome when the run didn't end, and that each returned with no error. */ static void -txn_spill_tests_one_version(uint32 line, uint32 col, uint32 unwrap_col, HostIoIo *io, const HostDeltaHold *hold, - HostSharedRecord state, uint64 number, bool numbered) +txn_spill_tests_returned(const SimcReport *report, uint32 line, uint32 col, const StdFmtPiece lead[2], + uint32 all_line) { - static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; - StdFmtArg args[1] = {{&number, std_u64_Display_fmt}}; - StdFmtArguments message = {pieces, lengthof(pieces), args}; - StdVec versions; - HostError error; - size_t one = 1; - uint8 key[8]; + static const StdFmtPiece outcome[] = {STD_FMT_ARG(0)}; + SimcEnding returned = {.tag = SIMC_ENDING_RETURNED}; + StdFmtArg args[1] = {{&report->outcome, simc_Outcome_Debug_fmt}}; + bool all = true; - std_u64_to_be_bytes(number, key); - if (!txn_memtable_table_versions(io, hold, state, key, sizeof(key), &versions, &error)) - STD_RESULT_EXPECT_FAILED(line, unwrap_col, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (versions.len != 1) - std_assert_failed(RUST_FILE, line, col, "==", &versions.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt, - numbered ? &message : NULL); - txn_spill_tests_versions_free(&versions); + if (report->outcome.tag != SIMC_OUTCOME_ENDED) + { + StdFmtArguments message = {lead, 2, args}; + + std_panic_fmt(RUST_FILE, line, col, &message); + } + for (size_t k = 0; all && k < report->outcome.u.ended.len; k++) + all = simc_Ending_PartialEq_eq(&STD_VEC_AT(&report->outcome.u.ended, simc_TupleStringEnding_t, k).f1, + &returned); + if (!all) + { + StdFmtArguments message = {outcome, lengthof(outcome), args}; + + std_panic_fmt(RUST_FILE, all_line, 9, &message); + } } -/* txn_spill_tests_doubled returns the scan case's key of a number: its double, as big-endian bytes, as its key closure does. */ -static void -txn_spill_tests_doubled(uint32 number, StdVec *result) +/* txn_spill_tests_fill takes the delta's pages of six kilobytes until it refuses one, as the spill cases' fillers do. */ +static bool +txn_spill_tests_fill(HostIoIo *io, StdVec *pages, HostError *error) { - uint8 be[4]; + HostDeltaPageSize size = {.bytes = 6 << 10, .words = 256}; - std_u32_to_be_bytes(number * 2, be); - std_slice_to_vec_Checked(result, be, sizeof(be), 1, 1); + for (;;) + { + StdVec taken; + HostDeltaFull full; + bool ok; + + if (!host_delta_alloc(io, &size, 1, &ok, &taken, &full, error)) + return false; + if (!ok) + return true; + std_vec_extend_from_slice_Checked(pages, taken.ptr, taken.len, sizeof(HostDeltaOwned), + _Alignof(HostDeltaOwned)); + std_vec_free_Checked(&taken, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + } } -/* - * Bugs row 74's scan: a scan's steps over a storage's rows merge its - * memtable's keys and its spilled keys in key order, both ways, across a - * spill file's chunks, and a key's lookup finds its spilled version in any - * chunk. - */ -void -test_txn_spill_tests_a_scan_merges_spilled_keys_across_chunks_in_order(void) +/* txn_spill_tests_one_range_writer is the one-range case's writer: it writes its few rows, spills them at the filled budget, and ends its writes. */ +static bool +txn_spill_tests_one_range_writer(void *captures, HostIoIo *io, HostError *error) { - static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; - static const TxnMemtableTableDirection ways[] = {TXN_MEMTABLE_TABLE_DIRECTION_FORWARD, - TXN_MEMTABLE_TABLE_DIRECTION_BACKWARD}; - static const uint32 looked[] = {0, 77, 150, 299}; - SimcTestIo test_io = simc_io(); - HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9744); - HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9744); - uint64 mine = simc_begin().f0; - uint64 other = simc_begin().f0; - uint8 odd[4]; - uint8 three[4]; + HostStorage storage = txn_memtable_testing_storage(9488); uint8 value[1000]; - StdVec expected; - StdVec odd_key; - StdVec versions; + HostSharedHeld held; TxnMemtableOp op; - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(mine), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - TxnMemtableVersionHandle handle; - HostDeltaHold hold; - HostError error; - bool sorted_odd = false; - size_t one = 1; + uint64 xid; + bool ok = true; bool spilled; - for (uint32 number = 0; number < 300; number++) + if (!txn_memtable_testing_stated(io, 9488, &held, error)) + return false; + xid = simc_begin().f0; + memset(value, 7, sizeof(value)); + op = txn_memtable_testing_put(value, sizeof(value)); + for (uint64 number = 0; ok && number < 5; number++) { - StdVec key; + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + size_t need = txn_memtable_version_encoded(&change.writer, change.op, change.shown, change.has_lsn, + change.lsn); + TxnMemtableVersionHandle handle; + uint64 touched; + uint8 key[8]; - memset(value, (uint8) number, sizeof(value)); - op = txn_memtable_testing_put(value, sizeof(value)); - txn_spill_tests_doubled(number, &key); - if (!txn_memtable_table_push(io, state, key.ptr, key.len, &change, &handle, &error)) - STD_RESULT_EXPECT_FAILED(2071, 59, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - std_vec_free_Checked(&key, 1, 1); - txn_memtable_Op_drop(&op); + std_u64_to_be_bytes(number, key); + ok = txn_written_touched(io, xid, &storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), need, &touched, error) && + txn_memtable_table_push(io, host_shared_Held_record(&held), key, sizeof(key), &change, &handle, error); } - std_u32_to_be_bytes(301, odd); - op = txn_memtable_testing_put((const uint8 *) "o", 1); - change.writer = txn_memtable_testing_writer(other); - if (!txn_memtable_table_push(io, state, odd, sizeof(odd), &change, &handle, &error)) - STD_RESULT_EXPECT_FAILED(2076, 47, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); + ok = ok && host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-written", error) && + txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, error); + if (ok) + STD_ASSERT(2368, 13, spilled, "spilled(io, &storage(9488), Of::Rows, xid)?"); + ok = ok && host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-spilled", error) && + txn_written_ended(io, error); txn_memtable_Op_drop(&op); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2077, 61, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2077, 9, spilled, "spilled(io, &storage(9744), Of::Rows, mine).unwrap()"); - std_vec_with_capacity_Checked(&expected, 301, sizeof(StdVec), _Alignof(StdVec)); - for (uint32 number = 0; number < 300; number++) - { - StdVec key; - - txn_spill_tests_doubled(number, &key); - if (!sorted_odd && memcmp(odd, key.ptr, sizeof(odd)) < 0) - { - std_slice_to_vec_Checked(&odd_key, odd, sizeof(odd), 1, 1); - std_vec_push_Checked(&expected, &odd_key, sizeof(StdVec), _Alignof(StdVec)); - sorted_odd = true; - } - std_vec_push_Checked(&expected, &key, sizeof(StdVec), _Alignof(StdVec)); - } - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2081, 36, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - for (size_t w = 0; w < lengthof(ways); w++) - { - StdVec keys; - StdVec after = {0}; - bool has_after = false; - - std_vec_new(&keys, sizeof(StdVec), _Alignof(StdVec)); - for (;;) - { - txn_TupleVecU8VecVersion_t found; - StdVec copy; - bool some; + host_shared_Held_drop(&held); + return ok; +} - if (!txn_memtable_table_next(io, &hold, state, has_after, after.ptr, after.len, ways[w], &some, &found, - &error)) - STD_RESULT_EXPECT_FAILED(2085, 102, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!some) - break; - STD_ASSERT_EQ(2086, 17, found.f1.len == 1, &found.f1.len, std_usize_Debug_fmt, &one, - std_usize_Debug_fmt); - txn_spill_tests_versions_free(&found.f1); - std_vec_clone(©, &found.f0, 1, 1); - std_vec_push_Checked(&keys, ©, sizeof(StdVec), _Alignof(StdVec)); - if (has_after) - std_vec_free_Checked(&after, 1, 1); - after = found.f0; - has_after = true; - } - if (has_after) - std_vec_free_Checked(&after, 1, 1); - if (ways[w] == TXN_MEMTABLE_TABLE_DIRECTION_BACKWARD) - for (size_t k = 0; k < keys.len / 2; k++) - { - StdVec swap = STD_VEC_AT(&keys, StdVec, k); +/* txn_spill_tests_filler is the one-range case's filler: it fills the delta, waits at a pause, and frees its pages. */ +static bool +txn_spill_tests_filler(void *captures, HostIoIo *io, HostError *error) +{ + StdVec pages; + bool ok; - STD_VEC_AT(&keys, StdVec, k) = STD_VEC_AT(&keys, StdVec, keys.len - 1 - k); - STD_VEC_AT(&keys, StdVec, keys.len - 1 - k) = swap; - } - if (!txn_memtable_table_tests_keys_eq(&keys, &expected)) - { - StdFmtArg args[1] = {{&ways[w], txn_spill_tests_Direction_Debug_fmt}}; - StdFmtArguments message = {pieces, lengthof(pieces), args}; + std_vec_new(&pages, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + ok = txn_spill_tests_fill(io, &pages, error) && + host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-filled", error); + std_vec_drop_Checked(&pages, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned), host_delta_Owned_drop); + return ok; +} - std_assert_failed(RUST_FILE, 2093, 13, "==", &keys, txn_memtable_table_tests_keys_Debug_fmt, &expected, - txn_memtable_table_tests_keys_Debug_fmt, &message); - } - txn_memtable_table_tests_keys_free(&keys); - } - txn_memtable_table_tests_keys_free(&expected); - for (size_t k = 0; k < lengthof(looked); k++) - { - StdVec key; - TxnMemtableVersionRecord record; - uint64 xid; +/* + * Bugs row 124's one-range tree: a tree of a few rows, its one range the + * memtable's floor, spills at a delta another process filled; its manifest + * takes none of the delta (bugs row 143), so the spill lists it at once. + */ +void +test_txn_spill_tests_a_one_range_tree_spills_at_a_full_budget(void) +{ + static const StdFmtPiece lead[] = {STD_FMT_LIT("the spill didn't link: "), STD_FMT_ARG(0)}; + SimcStep steps[2] = {{"filler", 6, PG_UINT64_MAX}, {"writer", 6, PG_UINT64_MAX}}; + SimcActor writer = {txn_spill_tests_one_range_writer}; + SimcActor filler = {txn_spill_tests_filler}; + SimcTestIo test_io; + HostIoIo *io; + HostSharedHeld held; + SimcRun run; + SimcReport report; + HostDeltaHold hold; + HostError error; - txn_spill_tests_doubled(looked[k], &key); - if (!txn_memtable_table_versions(io, &hold, state, key.ptr, key.len, &versions, &error)) - STD_RESULT_EXPECT_FAILED(2096, 76, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - std_vec_free_Checked(&key, 1, 1); - STD_ASSERT_EQ(2097, 13, versions.len == 1, &versions.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt); - record = txn_memtable_version_Version_record(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, 0)); - xid = txn_memtable_version_Record_writer(&record).xid; - STD_ASSERT_EQ(2098, 13, xid == mine, &xid, std_u64_Debug_fmt, &mine, std_u64_Debug_fmt); - txn_spill_tests_versions_free(&versions); - } - std_u32_to_be_bytes(3, three); - if (!txn_memtable_table_versions(io, &hold, state, three, sizeof(three), &versions, &error)) - STD_RESULT_EXPECT_FAILED(2100, 72, "called `Result::unwrap()` on an `Err` value", &error, + simc_size_delta(2 << 20); + test_io = simc_io(); + io = &test_io.io; + held = txn_memtable_testing_created(io, 9488); + host_shared_Held_drop(&held); + run = simc_Run_new(1); + simc_Run_actor(&run, "writer", 6, writer); + simc_Run_actor(&run, "filler", 6, filler); + simc_Run_until(&run, "writer", 6, "doltgres-spill-case-written"); + simc_Run_until(&run, "filler", 6, "doltgres-spill-case-filled"); + simc_Run_until(&run, "writer", 6, "doltgres-spill-case-spilled"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_spill_tests_returned(&report, 2387, 13, lead, 2389); + if (!txn_memtable_testing_stated(io, 9488, &held, &error)) + STD_RESULT_EXPECT_FAILED(2390, 37, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT(2100, 9, versions.len == 0, "table::versions(io, &hold, state, &3u32.to_be_bytes()).unwrap().is_empty()"); - txn_spill_tests_versions_free(&versions); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2391, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (uint64 number = 0; number < 5; number++) + txn_spill_tests_one_version(2393, 13, 89, io, &hold, host_shared_Held_record(&held), number, false); host_delta_Hold_drop(&hold); host_shared_Held_drop(&held); + simc_Report_drop(&report); simc_TestIo_drop(&test_io); } -#include "doltgres/txn/values.h" - -/* txn_spill_tests_newest_is asserts at its line that a value id's newest entry, in a state's value map or in its spilled value maps, is a wanted one or nothing. */ +/* txn_spill_tests_tupled_rows writes a transaction's rows of a range of numbers to a state's memtable, each counted in its written trees with no size and each a put whose tuple id is its number plus one. */ static void -txn_spill_tests_newest_is(uint32 line, uint32 unwrap_col, HostIoIo *io, HostSharedRecord state, bool spilled, - uint32 id, bool has_expected, const TxnMemtableMapEntry *expected) -{ - TxnMemtableMapEntry entry; - HostError error; - bool some; - bool ok; - StdOptionRef left; - StdOptionRef right = {has_expected, STD_ITEM(expected, txn_memtable_map_tests_seen_Debug_fmt)}; - - if (spilled) - ok = txn_spill_newest_value(io, state, id, &some, &entry, &error); - else - ok = txn_memtable_map_newest(io, state, TXN_MEMTABLE_MAP_MAP_VALUES, id, &some, &entry, &error); - if (!ok) - STD_RESULT_EXPECT_FAILED(line, unwrap_col, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - left = (StdOptionRef) - { - some, STD_ITEM(&entry, txn_memtable_map_tests_seen_Debug_fmt) - }; - STD_ASSERT_EQ(line, 9, some == has_expected && (!some || txn_memtable_map_tests_seen_eq(&entry, expected)), - &left, std_option_Debug_fmt, &right, std_option_Debug_fmt); - if (some) - txn_memtable_map_tests_seen_drop(&entry); -} - -/* txn_spill_tests_mark_Debug_fmt is a value map's mark's Debug, the tuple (u64, u64). */ -static bool -txn_spill_tests_mark_Debug_fmt(const void *value, StdFormatter *f) -{ - const std_TupleU64U64_t *mark = value; - StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&mark->f0, std_u64_Debug_fmt), STD_ITEM(&mark->f1, std_u64_Debug_fmt)); - - return std_tuple_Debug_fmt(&tuple, f); -} - -/* - * Bugs row 74's value map: a spill of a storage's value map takes out the - * entries only its transaction wrote, and a reader finds a spilled entry's - * newest below the memtable, a removal included, and moves the state's mark. - */ -void -test_txn_spill_tests_a_spilled_value_maps_entries_are_read_below_the_memtable(void) +txn_spill_tests_tupled_rows(uint32 touch_line, uint32 touch_col, uint32 push_line, uint32 push_col, HostIoIo *io, + HostSharedRecord state, const HostStorage *storage, uint64 xid, uint64 from, uint64 to) { - SimcTestIo test_io = simc_io(); - HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9745); - HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9745); - uint64 mine = simc_begin().f0; - uint64 other = simc_begin().f0; - const struct - { - uint64 xid; - uint32 id; - const char *value; - } pushes[] = {{mine, 7, "e"}, {mine, 8, "f"}, {mine, 8, NULL}, {other, 9, "g"}}; - TxnMemtableMapEntry e = {.has_value = true}; - TxnMemtableMapEntry removal = {.has_value = false}; - TxnMemtableMapEntry g = {.has_value = true}; - std_TupleU64U64_t before; - std_TupleU64U64_t after; HostError error; - bool spilled; - for (size_t k = 0; k < lengthof(pushes); k++) + for (uint64 number = from; number < to; number++) { - TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_DELETE}; - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; TxnMemtableVersionHandle handle; - uint8 key[4]; + uint64 touched; + uint8 key[8]; - if (pushes[k].value != NULL) - op = txn_memtable_testing_put((const uint8 *) pushes[k].value, 1); - std_u32_to_be_bytes(pushes[k].id, key); - if (!txn_memtable_table_alone_State(io, state, true, TXN_MEMTABLE_MAP_MAP_VALUES, key, sizeof(key), &change, - &handle, &error)) - STD_RESULT_EXPECT_FAILED(2122, 84, "called `Result::unwrap()` on an `Err` value", &error, + std_u64_to_be_bytes(number, key); + if (!txn_written_touched(io, xid, storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), 0, &touched, &error)) + STD_RESULT_EXPECT_FAILED(touch_line, touch_col, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + op.u.put.tuple.f0 = number + 1; + if (!txn_memtable_table_push(io, state, key, sizeof(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(push_line, push_col, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); txn_memtable_Op_drop(&op); } - before = txn_values_mark(state); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_VALUES, mine, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2125, 63, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2125, 9, spilled, "spilled(io, &storage(9745), Of::Values, mine).unwrap()"); - after = txn_values_mark(state); - if (after.f0 == before.f0 && after.f1 == before.f1) - std_assert_failed(RUST_FILE, 2126, 9, "!=", &after, txn_spill_tests_mark_Debug_fmt, &before, - txn_spill_tests_mark_Debug_fmt, NULL); - std_slice_to_vec_Checked(&e.value, (const uint8 *) "e", 1, 1, 1); - std_slice_to_vec_Checked(&g.value, (const uint8 *) "g", 1, 1, 1); - txn_spill_tests_newest_is(2127, 59, io, state, false, 7, false, NULL); - txn_spill_tests_newest_is(2128, 47, io, state, true, 7, true, &e); - txn_spill_tests_newest_is(2129, 47, io, state, true, 8, true, &removal); - txn_spill_tests_newest_is(2130, 47, io, state, true, 9, false, NULL); - txn_spill_tests_newest_is(2131, 59, io, state, false, 9, true, &g); - std_vec_free_Checked(&g.value, 1, 1); - std_vec_free_Checked(&e.value, 1, 1); - host_shared_Held_drop(&held); - simc_TestIo_drop(&test_io); } -#define TXN_MEMTABLE_TABLE_RS Unit -#define TXN_MEMTABLE_TABLE_RETURNS bool -#define TXN_MEMTABLE_TABLE_RESULT_PARAMS , HostError *error -#define TXN_MEMTABLE_TABLE_DECLARE -#include "doltgres/txn/memtable/table_template.h" +/* txn_spill_tests_kind_number_t is a spill file's kind and number, as the merge case lists them. */ +typedef struct txn_spill_tests_kind_number_t +{ + TxnRunsKind kind; + uint32 number; +} txn_spill_tests_kind_number_t; -/* txn_spill_tests_kinds_Debug_fmt is a Vec's Debug. */ +/* txn_spill_tests_kind_numbers_Debug_fmt is a Vec<(Kind, u32)>'s Debug. */ static bool -txn_spill_tests_kinds_Debug_fmt(const void *value, StdFormatter *f) +txn_spill_tests_kind_numbers_Debug_fmt(const void *value, StdFormatter *f) { const StdVec *kinds = value; StdDebugList list = std_fmt_debug_list(f); for (size_t k = 0; k < kinds->len; k++) - std_DebugList_entry(&list, &STD_VEC_AT(kinds, TxnRunsKind, k), txn_spill_tests_Kind_Debug_fmt); + { + const txn_spill_tests_kind_number_t *kind = &STD_VEC_AT(kinds, txn_spill_tests_kind_number_t, k); + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&kind->kind, txn_spill_tests_Kind_Debug_fmt), + STD_ITEM(&kind->number, std_u32_Debug_fmt)); + + std_DebugList_entry(&list, &tuple, std_tuple_Debug_fmt); + } return std_DebugList_finish(&list); } -/* txn_spill_tests_kinds_are asserts at its place that listed files are of a list of kinds, newest first. */ +/* txn_spill_tests_tuple_key_is asserts at its line that a number's row's tuple id, its number plus one, finds the number's key, the message the number. */ static void -txn_spill_tests_kinds_are(uint32 line, const StdVec *files, const TxnRunsKind *wanted, size_t wanted_len) +txn_spill_tests_tuple_key_is(uint32 line, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, + uint64 number) { - StdVec kinds; - StdVec wanted_vec = {(void *) wanted, wanted_len, wanted_len}; - bool eq = files->len == wanted_len; + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + HostRow tid = {number + 1}; + TxnMemtableMapEntry found; + TxnMemtableMapEntry key = {.has_value = true}; + HostError error; + uint8 be[8]; - std_vec_with_capacity_Checked(&kinds, files->len, sizeof(TxnRunsKind), _Alignof(TxnRunsKind)); - for (size_t k = 0; k < files->len; k++) - std_vec_push_Checked(&kinds, &STD_VEC_AT(files, TxnSpillFile, k).kind, sizeof(TxnRunsKind), - _Alignof(TxnRunsKind)); - for (size_t k = 0; eq && k < wanted_len; k++) - eq = STD_VEC_AT(&kinds, TxnRunsKind, k) == wanted[k]; - STD_ASSERT_EQ(line, 9, eq, &kinds, txn_spill_tests_kinds_Debug_fmt, &wanted_vec, txn_spill_tests_kinds_Debug_fmt); - std_vec_free_Checked(&kinds, sizeof(TxnRunsKind), _Alignof(TxnRunsKind)); + if (!txn_spill_tuple_key(io, hold, state, tid, &found.has_value, &found.value, &error)) + STD_RESULT_EXPECT_FAILED(line, 69, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_u64_to_be_bytes(number, be); + key.value = (StdVec) + { + be, sizeof(be), sizeof(be) + }; + if (!txn_memtable_map_tests_seen_eq(&found, &key)) + { + StdFmtArg args[1] = {{&number, std_u64_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_assert_failed(RUST_FILE, line, 13, "==", &found, txn_memtable_map_tests_seen_Debug_fmt, &key, + txn_memtable_map_tests_seen_Debug_fmt, &message); + } + txn_memtable_map_tests_seen_drop(&found); } -/* txn_spill_tests_tuple_is asserts at its line that a tid's row's key in a state's tuple id table, or in its spilled tuple ids, is a wanted key or nothing. */ -static void -txn_spill_tests_tuple_is(uint32 line, uint32 col, uint32 unwrap_col, HostIoIo *io, HostSharedRecord state, uint64 tid, - const TxnMemtableMapEntry *expected) +/* + * Bugs row 124's merge of tuple ids: a transaction's two spills of one tree, + * the second as large as the first, merge into one row file, and the tuple-id + * files their entries name merge into one of the same number, which the + * merged entry names; every row's tuple id still finds its key. + */ +void +test_txn_spill_tests_a_merge_of_row_spills_merges_the_tuple_ids_their_entries_name(void) { - HostRow row = {tid}; - TxnMemtableMapEntry found; + static const uint64 numbers[] = {0, 99, 100, 199}; + txn_spill_tests_kind_number_t wanted[] = {{TXN_RUNS_KIND_SPILLED_TUPLES, 2}, {TXN_RUNS_KIND_SPILLED_ROWS, 2}}; + StdVec wanted_vec = {wanted, lengthof(wanted), lengthof(wanted)}; + SimcTestIo test_io; + HostIoIo *io; + HostSharedHeld held; + HostSharedRecord state; + HostStorage storage = txn_memtable_testing_storage(9489); + StdVec files; + StdVec kinds; + HostDeltaHold hold; HostError error; + uint64 xid; + bool spilled; + bool eq; - if (!txn_memtable_tuples_get(io, state, row, &found.has_value, &found.value, &error)) - STD_RESULT_EXPECT_FAILED(line, unwrap_col, "called `Result::unwrap()` on an `Err` value", &error, + simc_size_delta(64 << 20); + test_io = simc_io(); + io = &test_io.io; + held = txn_memtable_testing_created(io, 9489); + state = host_shared_Held_record(&held); + xid = simc_begin().f0; + for (uint64 half = 0; half < 2; half++) + { + txn_spill_tests_tupled_rows(2412, 78, 2418, 55, io, state, &storage, xid, half * 100, half * 100 + 100); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2420, 64, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2420, 13, spilled, "spilled(io, &storage(9489), Of::Rows, xid).unwrap()"); + } + files = txn_spill_tests_files_of(io, state); + std_vec_with_capacity_Checked(&kinds, files.len, sizeof(txn_spill_tests_kind_number_t), + _Alignof(txn_spill_tests_kind_number_t)); + for (size_t k = 0; k < files.len; k++) + { + const TxnSpillFile *file = &STD_VEC_AT(&files, TxnSpillFile, k); + txn_spill_tests_kind_number_t kind = {file->kind, file->number}; + + std_vec_push_Checked(&kinds, &kind, sizeof(kind), _Alignof(txn_spill_tests_kind_number_t)); + } + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2424, 36, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT_EQ(line, col, txn_memtable_map_tests_seen_eq(&found, expected), &found, - txn_memtable_map_tests_seen_Debug_fmt, expected, txn_memtable_map_tests_seen_Debug_fmt); - txn_memtable_map_tests_seen_drop(&found); + eq = kinds.len == lengthof(wanted); + for (size_t k = 0; eq && k < lengthof(wanted); k++) + eq = STD_VEC_AT(&kinds, txn_spill_tests_kind_number_t, k).kind == wanted[k].kind && + STD_VEC_AT(&kinds, txn_spill_tests_kind_number_t, k).number == wanted[k].number; + STD_ASSERT_EQ(2425, 9, eq, &kinds, txn_spill_tests_kind_numbers_Debug_fmt, &wanted_vec, + txn_spill_tests_kind_numbers_Debug_fmt); + for (size_t k = 0; k < lengthof(numbers); k++) + txn_spill_tests_tuple_key_is(2428, io, &hold, state, numbers[k]); + host_delta_Hold_drop(&hold); + if (!txn_written_ended(io, &error)) + STD_RESULT_EXPECT_FAILED(2431, 28, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&kinds, sizeof(txn_spill_tests_kind_number_t), _Alignof(txn_spill_tests_kind_number_t)); + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); } /* - * Bugs row 74's tuple ids: a spill of a storage's rows takes its rows' tuple - * ids out of the tuple id table into a file of spilled tuple ids, and each - * tid still finds its row's key, the other writer's from the table. + * Bugs row 143's merge at a full budget: a transaction's two small spills of + * one tree at a delta another process filled merge into one file, listed in + * place of the two with no page of the delta, and every key reads back + * through it. */ void -test_txn_spill_tests_a_spills_tuple_ids_still_find_their_rows_keys(void) +test_txn_spill_tests_a_merge_at_a_full_budget_lists_its_file_in_place_of_its_sources(void) { - SimcTestIo test_io = simc_io(); - HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9746); - HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9746); - uint64 mine = simc_begin().f0; - uint64 other = simc_begin().f0; - const struct - { - uint64 xid; - uint64 number; - } writes[] = {{mine, 1}, {mine, 2}, {mine, 3}, {other, 4}}; + static const uint64 numbers[] = {0, 19}; static const TxnRunsKind wanted[] = {TXN_RUNS_KIND_SPILLED_TUPLES, TXN_RUNS_KIND_SPILLED_ROWS}; - TxnMemtableMapEntry none = {.has_value = false}; + SimcTestIo test_io; + HostIoIo *io; + HostSharedHeld held; + HostSharedRecord state; + HostStorage storage = txn_memtable_testing_storage(9490); + StdVec filler; + HostDeltaHold hold; StdVec files; HostError error; + uint64 xid; bool spilled; - for (size_t k = 0; k < lengthof(writes); k++) + simc_size_delta(2 << 20); + test_io = simc_io(); + io = &test_io.io; + held = txn_memtable_testing_created(io, 9490); + state = host_shared_Held_record(&held); + xid = simc_begin().f0; + std_vec_new(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + for (uint64 half = 0; half < 2; half++) { - uint64 tid = UINT64CONST(1) << 16 | writes[k].number; - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(writes[k].xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - txn_memtable_tuples_tests_stored_t captures = {.tid = tid, .change = &change}; - txn_TupleRefSliceU8Usize_t rows[1]; - uint8 key[8]; - - op.u.put.tuple.f0 = tid; - std_u64_to_be_bytes(writes[k].number, key); - rows[0].f0 = (StdBytes) - { - key, sizeof(key) - }; - rows[0].f1 = txn_memtable_version_encoded(&change.writer, change.op, change.shown, false, 0); - if (!txn_memtable_table_prepare_Unit(io, state, rows, lengthof(rows), NULL, 0, true, - txn_memtable_tuples_tests_stored_inside, &captures, NULL, &error)) - STD_RESULT_EXPECT_FAILED(2161, 14, "called `Result::unwrap()` on an `Err` value", &error, + txn_spill_tests_tupled_rows(2450, 78, 2456, 55, io, state, &storage, xid, half * 10, half * 10 + 10); + if ((filler.len > 0 || half * 10 > 0) && !txn_spill_tests_fill(io, &filler, &error)) + STD_RESULT_EXPECT_FAILED(2459, 96, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - txn_memtable_Op_drop(&op); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2463, 64, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2463, 13, spilled, "spilled(io, &storage(9490), Of::Rows, xid).unwrap()"); } - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2163, 61, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2163, 9, spilled, "spilled(io, &storage(9746), Of::Rows, mine).unwrap()"); files = txn_spill_tests_files_of(io, state); - txn_spill_tests_kinds_are(2165, &files, wanted, lengthof(wanted)); + txn_spill_tests_kinds_are(2466, &files, wanted, lengthof(wanted)); std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - for (uint64 number = 1; number <= 4; number++) + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2467, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; k < lengthof(numbers); k++) { - TxnMemtableMapEntry key = {.has_value = true}; - uint8 be[8]; - - std_u64_to_be_bytes(number, be); - std_slice_to_vec_Checked(&key.value, be, sizeof(be), 1, 1); - txn_spill_tests_tuple_is(2167, 13, 65, io, state, UINT64CONST(1) << 16 | number, &key); - std_vec_free_Checked(&key.value, 1, 1); + txn_spill_tests_one_version(2470, 13, 64, io, &hold, state, numbers[k], true); + txn_spill_tests_tuple_key_is(2471, io, &hold, state, numbers[k]); } - txn_spill_tests_tuple_is(2169, 9, 56, io, state, UINT64CONST(1) << 16 | 9, &none); + host_delta_Hold_drop(&hold); + std_vec_drop_Checked(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned), host_delta_Owned_drop); + if (!txn_written_ended(io, &error)) + STD_RESULT_EXPECT_FAILED(2474, 28, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); } -#include "doltgres/txn/fold.h" -#include "doltgres/txn/insert.h" +/* txn_spill_tests_merger is the merge window case's merger: its transaction's second spill, which merges with its first. */ +static bool +txn_spill_tests_merger(void *captures, HostIoIo *io, HostError *error) +{ + HostStorage storage = txn_memtable_testing_storage(9491); + bool spilled; -/* TXN_SPILL_TESTS_WRITERS are the three writers' names and databases. */ -static const struct + return txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, *(const uint64 *) captures, &spilled, error); +} + +/* txn_spill_tests_found_t is the merge window case's reader's finding: whether it read, and its count of the first key's versions. */ +typedef struct txn_spill_tests_found_t { - const char *name; - uint32 database; -} TXN_SPILL_TESTS_WRITERS[] = {{"w1", 9484}, {"w2", 9485}, {"w3", 9486}}; + bool some; + size_t count; +} txn_spill_tests_found_t; -/* txn_spill_tests_tree returns the three writers' case's storage of a tree of a database, as its trees closure numbers them. */ -static HostStorage -txn_spill_tests_tree(uint32 database, uint32 tree) +/* txn_spill_tests_found_Debug_fmt is the reader's finding's Debug, an Option. */ +static bool +txn_spill_tests_found_Debug_fmt(const void *value, StdFormatter *f) { - HostStorage storage = txn_memtable_testing_storage(database); + const txn_spill_tests_found_t *found = value; + StdOptionRef option = {found->some, STD_ITEM(&found->count, std_usize_Debug_fmt)}; - storage.relation = 16384 + tree; - return storage; + return std_option_Debug_fmt(&option, f); } -/* txn_spill_tests_based_t is the three writers' case's base of a tree: its repository, a shared read, and its state. */ -typedef struct txn_spill_tests_based_t -{ - RepoRepo repo; - HostSharedReader reading; - HostSharedHeld held; -} txn_spill_tests_based_t; - -/* txn_spill_tests_based opens a tree's repository and a shared read, and makes the tree's state, keyless and permanent, as the three writers' case's based closure does. */ +/* txn_spill_tests_window_reader is the merge window case's reader: it counts the first key's versions. */ static bool -txn_spill_tests_based(HostIoIo *io, const HostStorage *tree, txn_spill_tests_based_t *result, HostError *error) +txn_spill_tests_window_reader(void *captures, HostIoIo *io, HostError *error) { - TxnStateMade made = {.key = {.tag = TXN_BASE_KEY_NONE}, .table = tree->relation}; - ChunksErrorError failed; + txn_spill_tests_found_t *found = captures; + HostSharedHeld held; + HostDeltaHold hold; + StdVec versions; + uint8 key[8]; + bool ok; - if (!repo_Repo_open(io, tree->database, HOST_PG_DEFAULT, &result->repo, &failed)) - { - *error = txn_base_storage(&failed); + if (!txn_memtable_testing_stated(io, 9491, &held, error)) return false; - } - if (!host_shared_read(io, &result->reading, error)) + if (!host_delta_hold(io, &hold, error)) { - repo_Repo_drop(&result->repo); + host_shared_Held_drop(&held); return false; } - if (!txn_state_state(io, &result->repo, &result->reading, tree, TXN_STATE_PERSISTENCE_PERMANENT, true, &made, - &result->held, error)) + std_u64_to_be_bytes(0, key); + ok = txn_memtable_table_versions(io, &hold, host_shared_Held_record(&held), key, sizeof(key), &versions, error); + if (ok) { - host_shared_Reader_drop(&result->reading); - repo_Repo_drop(&result->repo); - return false; + *found = (txn_spill_tests_found_t) + { + true, versions.len + }; + txn_spill_tests_versions_free(&versions); } - return true; -} - -/* txn_spill_tests_based_drop drops a tree's base: its state, its read, then its repository. */ -static void -txn_spill_tests_based_drop(txn_spill_tests_based_t *based) -{ - host_shared_Held_drop(&based->held); - host_shared_Reader_drop(&based->reading); - repo_Repo_drop(&based->repo); -} - -/* txn_spill_tests_three_t is the three writers' case's shared state: its trees and rows each, whether it folds, its counts of writers finished and ended, the databases committed, and whether a writer failed. */ -typedef struct txn_spill_tests_three_t -{ - uint32 trees_each; - uint64 rows; - bool folding; - size_t finished; - size_t ended; - /* committed is a Vec. */ - StdVec committed; - bool failed; -} txn_spill_tests_three_t; - -/* txn_spill_tests_writing_t is a writer's captures in the three writers' case: the shared state and its database. */ -typedef struct txn_spill_tests_writing_t -{ - txn_spill_tests_three_t *three; - uint32 database; -} txn_spill_tests_writing_t; - -/* txn_spill_tests_states_free drops a writer's trees' states and frees them. */ -static void -txn_spill_tests_states_free(StdVec *states) -{ - for (size_t k = 0; k < states->len; k++) - host_shared_Held_drop(&STD_VEC_AT(states, HostSharedHeld, k)); - std_vec_free_Checked(states, sizeof(HostSharedHeld), _Alignof(HostSharedHeld)); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&held); + return ok; } -/* txn_spill_tests_pushed writes each row of a writer's trees, a key at a time across its trees, each counted in the transaction's written trees first. */ -static bool -txn_spill_tests_pushed(const txn_spill_tests_writing_t *c, HostIoIo *io, const StdVec *states, HostXid xid, - const TxnMemtableOp *op, HostError *error) +/* + * Bugs row 124's merge window: a reader that read the slots before a merge + * pended its file, and lists the chain after the merge unchained its sources + * but before it linked the merged file, sees the chain changed, starts over, + * and reads the key's version once, through the pended merged file. + */ +void +test_txn_spill_tests_a_reader_across_a_merges_unchain_reads_each_version_once(void) { - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid.f0), .op = op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - size_t need = txn_memtable_version_encoded(&change.writer, change.op, change.shown, change.has_lsn, - change.lsn); + static const StdFmtPiece lead[] = {STD_FMT_LIT("the run didn't end: "), STD_FMT_ARG(0)}; + SimcStep steps[2] = {{"reader", 6, PG_UINT64_MAX}, {"merger", 6, PG_UINT64_MAX}}; + txn_spill_tests_found_t found = {0}; + txn_spill_tests_found_t once = {true, 1}; + SimcTestIo test_io; + HostIoIo *io; + HostSharedHeld held; + HostStorage storage = txn_memtable_testing_storage(9491); + SimcActor merger = {txn_spill_tests_merger}; + SimcActor reader = {txn_spill_tests_window_reader, &found}; + SimcRun run; + SimcReport report; + HostError error; + uint64 xid; + bool spilled; - for (uint64 number = 0; number < c->three->rows; number++) + simc_size_delta(64 << 20); + test_io = simc_io(); + io = &test_io.io; + held = txn_memtable_testing_created(io, 9491); + xid = simc_begin().f0; + for (uint64 half = 0; half < 2; half++) { - uint8 key[8]; - - std_u64_to_be_bytes(number, key); - for (size_t k = 0; k < states->len; k++) + for (uint64 number = half * 10; number < half * 10 + 10; number++) { - HostStorage tree = txn_spill_tests_tree(c->database, (uint32) k); + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; TxnMemtableVersionHandle handle; uint64 touched; + uint8 key[8]; - if (!txn_written_touched(io, xid.f0, &tree, TXN_WRITTEN_OF_ROWS, key, sizeof(key), need, &touched, - error) || - !txn_memtable_table_push(io, host_shared_Held_record(&STD_VEC_AT(states, HostSharedHeld, k)), key, - sizeof(key), &change, &handle, error)) - return false; + std_u64_to_be_bytes(number, key); + if (!txn_written_touched(io, xid, &storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), 0, &touched, &error)) + STD_RESULT_EXPECT_FAILED(2491, 78, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_memtable_table_push(io, host_shared_Held_record(&held), key, sizeof(key), &change, &handle, + &error)) + STD_RESULT_EXPECT_FAILED(2499, 63, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + } + if (half == 0) + { + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2502, 68, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2502, 17, spilled, "spilled(io, &storage(9491), Of::Rows, xid).unwrap()"); } } - return true; + host_shared_Held_drop(&held); + merger.captures = &xid; + run = simc_Run_new(1); + simc_Run_actor(&run, "merger", 6, merger); + simc_Run_actor(&run, "reader", 6, reader); + simc_Run_until(&run, "merger", 6, "doltgres-spill-linked"); + simc_Run_until(&run, "reader", 6, "doltgres-spill-slots-read"); + simc_Run_until(&run, "merger", 6, "doltgres-spill-merge-unchained"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_spill_tests_returned(&report, 2521, 67, lead, 2522); + STD_ASSERT_EQ(2523, 9, found.some && found.count == 1, &found, txn_spill_tests_found_Debug_fmt, &once, + txn_spill_tests_found_Debug_fmt); + if (!txn_written_ended(io, &error)) + STD_RESULT_EXPECT_FAILED(2524, 28, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_Report_drop(&report); + simc_TestIo_drop(&test_io); } -/* txn_spill_tests_wrote is a writer's body in the three writers' case, its wrote closure: it writes its trees' rows, waits for the others or commits, and ends its writes. */ +/* txn_spill_tests_paused_wrote writes a transaction's rows of a count to each of a database's nine trees, a tree at a time, as the paused holder case's wrote closure does. */ static bool -txn_spill_tests_wrote(const txn_spill_tests_writing_t *c, HostIoIo *io, HostError *error) +txn_spill_tests_paused_wrote(HostIoIo *io, uint32 database, uint64 rows, HostError *error) { - txn_spill_tests_three_t *three = c->three; + uint64 xid = simc_begin().f0; uint8 value[1000]; - StdVec states; - HostXid xid; TxnMemtableOp op; - bool ok; + bool ok = true; - std_vec_new(&states, sizeof(HostSharedHeld), _Alignof(HostSharedHeld)); - for (uint32 tree = 0; tree < three->trees_each; tree++) + memset(value, 7, sizeof(value)); + op = txn_memtable_testing_put(value, sizeof(value)); + for (uint32 tree = 0; ok && tree < 9; tree++) { - HostStorage storage = txn_spill_tests_tree(c->database, tree); - HostSharedHeld held; + HostStorage storage = txn_spill_tests_tree(database, tree); + HostSharedHeld state; - if (!txn_memtable_testing_held(io, &storage, &held, error)) + if (!txn_memtable_testing_held(io, &storage, &state, error)) { - txn_spill_tests_states_free(&states); - return false; + ok = false; + break; } - std_vec_push_Checked(&states, &held, sizeof(held), _Alignof(HostSharedHeld)); - } - xid = simc_begin(); - memset(value, 7, sizeof(value)); - op = txn_memtable_testing_put(value, sizeof(value)); - ok = txn_spill_tests_pushed(c, io, &states, xid, &op, error); - txn_memtable_Op_drop(&op); - txn_spill_tests_states_free(&states); - if (!ok) - return false; - three->finished++; - while (!three->folding && three->finished < lengthof(TXN_SPILL_TESTS_WRITERS)) - { - if ((size_t) simc_delta_waiting() + three->finished >= lengthof(TXN_SPILL_TESTS_WRITERS)) + for (uint64 number = 0; ok && number < rows; number++) { - *error = txn_spill_internal("every writer still writing sleeps at the budget"); - return false; + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + size_t need = txn_memtable_version_encoded(&change.writer, change.op, change.shown, + change.has_lsn, change.lsn); + TxnMemtableVersionHandle handle; + uint64 touched; + uint8 key[8]; + + std_u64_to_be_bytes(number, key); + ok = txn_written_touched(io, xid, &storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), need, &touched, + error) && + txn_memtable_table_push(io, host_shared_Held_record(&state), key, sizeof(key), &change, &handle, + error); } - if (!host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-written", error)) - return false; + host_shared_Held_drop(&state); } - if (three->folding) - { - TxnStatusOutcome committed = {.tag = TXN_STATUS_OUTCOME_COMMITTED}; - TxnStatusRoomed roomed; - bool took; + txn_memtable_Op_drop(&op); + return ok; +} - if (!host_wal_insert(HOST_IO_IO_IO_DOORS(io), HOST_WAL_KIND_ROW_CHANGE, NULL, 0, &committed.u.committed, - error) || - !txn_status_take(io, xid, committed, &roomed, &took, error)) - return false; - simc_commit(xid); - std_vec_push_Checked(&three->committed, &c->database, sizeof(uint32), _Alignof(uint32)); - } - if (!txn_written_ended(io, error)) +/* txn_spill_tests_holder is the paused holder case's holder: it writes its trees, waits at a pause, and ends its writes. */ +static bool +txn_spill_tests_holder(void *captures, HostIoIo *io, HostError *error) +{ + return txn_spill_tests_paused_wrote(io, 9492, 190, error) && + host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-held", error) && + txn_written_ended(io, error); +} + +/* txn_spill_tests_relieving_writer is the paused holder case's writer: its relief spills the paused transaction's versions. */ +static bool +txn_spill_tests_relieving_writer(void *captures, HostIoIo *io, HostError *error) +{ + bool relieved; + + if (!txn_spill_relieved(io, &relieved, error)) return false; - three->ended++; + STD_ASSERT_MSG(2570, 13, relieved, "the paused transaction's versions spill"); return true; } -/* txn_spill_tests_writer is a writer's actor in the three writers' case: its body, a failure marked in the shared state. */ -static bool -txn_spill_tests_writer(void *captures, HostIoIo *io, HostError *error) +/* + * Bugs row 144's paused holder: a transaction that wrote trees each under the + * spill threshold waits at a pause before its commit, not at the budget, so + * it spills nothing itself; a process with nothing of its own to spill + * relieves the budget by spilling the paused transaction's versions from + * every tree, and the paused one's rows read back after its commit. + */ +void +test_txn_spill_tests_a_process_with_nothing_of_its_own_spills_a_paused_transactions_versions(void) { - const txn_spill_tests_writing_t *c = captures; - bool wrote = txn_spill_tests_wrote(c, io, error); + static const StdFmtPiece lead[] = {STD_FMT_LIT("the writer didn't finish: "), STD_FMT_ARG(0)}; + SimcStep steps[2] = {{"writer", 6, PG_UINT64_MAX}, {"holder", 6, PG_UINT64_MAX}}; + SimcActor holder = {txn_spill_tests_holder}; + SimcActor writer = {txn_spill_tests_relieving_writer}; + SimcTestIo test_io; + HostIoIo *io; + HostSharedHeld held; + SimcRun run; + SimcReport report; + HostDeltaHold hold; + HostError error; - c->three->failed |= !wrote; - return wrote; + simc_size_delta(2 << 20); + test_io = simc_io(); + io = &test_io.io; + held = txn_memtable_testing_created(io, 9492); + host_shared_Held_drop(&held); + run = simc_Run_new(1); + simc_Run_actor(&run, "holder", 6, holder); + simc_Run_actor(&run, "writer", 6, writer); + simc_Run_until(&run, "holder", 6, "doltgres-spill-case-held"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_spill_tests_returned(&report, 2577, 13, lead, 2579); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2580, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (uint32 tree = 0; tree < 9; tree++) + { + HostStorage storage = txn_spill_tests_tree(9492, tree); + HostSharedHeld state; + StdVec files; + + if (!txn_memtable_testing_held(io, &storage, &state, &error)) + STD_RESULT_EXPECT_FAILED(2582, 41, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + files = txn_spill_tests_files_of(io, host_shared_Held_record(&state)); + STD_ASSERT_MSG(2583, 13, files.len > 0, "every tree spilled"); + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + txn_spill_tests_one_version(2585, 17, 94, io, &hold, host_shared_Held_record(&state), 0, false); + txn_spill_tests_one_version(2585, 17, 94, io, &hold, host_shared_Held_record(&state), 189, false); + host_shared_Held_drop(&state); + } + host_delta_Hold_drop(&hold); + simc_Report_drop(&report); + simc_TestIo_drop(&test_io); } -/* txn_spill_tests_folded folds each tree of each database committed, as the three writers' case's folder does when a writer waits. */ +/* txn_spill_tests_owner is the turn case's owner: it writes its rows, and its own spill at the pause spills nothing on the helper's turn. */ static bool -txn_spill_tests_folded(const txn_spill_tests_three_t *three, HostIoIo *io, const StdVec *all, HostError *error) +txn_spill_tests_owner(void *captures, HostIoIo *io, HostError *error) { - const TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; - const TxnInsertPage page = {.bytes = 8168, .most = 291}; + HostStorage storage = txn_memtable_testing_storage(9494); + HostSharedHeld state; + uint64 xid; + bool ok = true; + bool spilled; - for (size_t k = 0; k < all->len; k++) - for (uint32 tree = 0; tree < three->trees_each; tree++) - { - HostStorage storage = txn_spill_tests_tree(STD_VEC_AT(all, uint32, k), tree); - txn_spill_tests_based_t based; - StdVec warned; - HostLsn point; - bool has_point; - bool folded; - bool ok; + if (!txn_memtable_testing_stated(io, 9494, &state, error)) + return false; + xid = simc_begin().f0; + for (uint64 number = 0; ok && number < 50; number++) + { + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + size_t need = txn_memtable_version_encoded(&change.writer, change.op, change.shown, change.has_lsn, + change.lsn); + TxnMemtableVersionHandle handle; + uint64 touched; + uint8 key[8]; - if (!txn_spill_tests_based(io, &storage, &based, error)) - return false; - std_vec_new(&warned, sizeof(StdString), _Alignof(StdString)); - ok = txn_fold_point(io, &has_point, &point, error) && - txn_fold_fold_now(io, &based.repo, &based.reading, &based.held, &order, false, has_point, point, - PG_UINT64_MAX, &warned, &page, NULL, &folded, error); - std_vec_drop_Checked(&warned, sizeof(StdString), _Alignof(StdString), std_String_drop); - txn_spill_tests_based_drop(&based); - if (!ok) - return false; - } - return true; + std_u64_to_be_bytes(number, key); + ok = txn_written_touched(io, xid, &storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), need, &touched, error) && + txn_memtable_table_push(io, host_shared_Held_record(&state), key, sizeof(key), &change, &handle, error); + txn_memtable_Op_drop(&op); + } + ok = ok && host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-written", error) && + txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, error); + if (ok) + STD_ASSERT_MSG(2615, 13, !spilled, "the owner spills nothing on the helper's turn"); + ok = ok && txn_written_ended(io, error); + host_shared_Held_drop(&state); + return ok; } -/* txn_spill_tests_folder is the three writers' case's folder: while a writer is unended, it folds the committed databases' trees whenever a writer waits at the budget and a database committed since, or every five thousand idle steps. */ +/* txn_spill_tests_helper is the turn case's helper: its relief spills the owner's transaction's versions. */ static bool -txn_spill_tests_folder(void *captures, HostIoIo *io, HostError *error) +txn_spill_tests_helper(void *captures, HostIoIo *io, HostError *error) { - txn_spill_tests_three_t *three = captures; - size_t folded = 0; - int32 idle = 0; - - while (three->ended < lengthof(TXN_SPILL_TESTS_WRITERS)) - { - StdVec all; - bool fresh; - bool ok = true; + bool relieved; - if (three->failed) - { - *error = txn_spill_internal("a writer failed, so the fold has nothing to wait for"); - return false; - } - std_vec_clone(&all, &three->committed, sizeof(uint32), _Alignof(uint32)); - fresh = all.len > folded; - idle = !fresh && simc_delta_waiting() > 0 ? idle + 1 : 0; - if (idle > 100000) - { - std_vec_free_Checked(&all, sizeof(uint32), _Alignof(uint32)); - *error = txn_spill_internal("every writer still writing sleeps with nothing committed to fold"); - return false; - } - if (simc_delta_waiting() > 0 && (fresh || idle % 5000 == 4999)) - { - folded = all.len; - ok = txn_spill_tests_folded(three, io, &all, error); - } - std_vec_free_Checked(&all, sizeof(uint32), _Alignof(uint32)); - if (!ok || !host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-folding", error)) - return false; - } + if (!txn_spill_relieved(io, &relieved, error)) + return false; + STD_ASSERT(2619, 13, relieved, "relieved(io)?"); return true; } /* - * txn_spill_tests_three_writers runs bugs row 124's three writers, each of a - * count of trees of a count of rows: holding their commits until all three - * have written, or, folding, each committing once written while a fold runs, - * its commit at an empty record's end, since the simulation writes no commit - * record, so the fold's point passes it. + * Bugs row 144's turn: while a helper spills a transaction's versions, held + * inside its spill, the transaction's owner asks to spill its own tree and + * spills nothing, the turn being the helper's; once the helper is done, the + * transaction's files have distinct numbers and every key reads once. */ -static void -txn_spill_tests_three_writers(uint32 trees_each, uint64 rows, bool folding) +void +test_txn_spill_tests_an_owner_spills_nothing_while_a_helper_holds_its_transactions_turn(void) { - static const StdFmtPiece unfinished[] = {STD_FMT_LIT("the writers didn't finish, "), STD_FMT_ARG(0), - STD_FMT_LIT(" row and "), STD_FMT_ARG(1), STD_FMT_LIT(" tuple-id entries in "), STD_FMT_ARG(2), - STD_FMT_LIT(" bytes: "), STD_FMT_ARG(3)}; - static const StdFmtPiece outcome[] = {STD_FMT_ARG(0)}; - txn_spill_tests_three_t three = {.trees_each = trees_each, .rows = rows, .folding = folding}; - txn_spill_tests_writing_t writing[lengthof(TXN_SPILL_TESTS_WRITERS)]; - SimcEnding returned = {.tag = SIMC_ENDING_RETURNED}; + static const StdFmtPiece lead[] = {STD_FMT_LIT("the run didn't end: "), STD_FMT_ARG(0)}; + SimcStep steps[2] = {{"owner", 5, PG_UINT64_MAX}, {"helper", 6, PG_UINT64_MAX}}; + SimcActor owner = {txn_spill_tests_owner}; + SimcActor helper = {txn_spill_tests_helper}; SimcTestIo test_io; HostIoIo *io; + HostSharedHeld held; SimcRun run; SimcReport report; HostDeltaHold hold; HostError error; - bool all; + StdVec files; + size_t count; + size_t distinct = 0; - simc_size_delta(2 << 20); + simc_size_delta(64 << 20); test_io = simc_io(); io = &test_io.io; - for (size_t w = 0; w < lengthof(TXN_SPILL_TESTS_WRITERS); w++) - { - RepoRepo repo; - ChunksErrorError failed; - - if (!repo_Repo_create(io, TXN_SPILL_TESTS_WRITERS[w].database, HOST_PG_DEFAULT, &repo, &failed)) - STD_RESULT_EXPECT_FAILED(2215, 84, "called `Result::unwrap()` on an `Err` value", &failed, - chunks_error_Error_Debug_fmt); - repo_Repo_drop(&repo); - for (uint32 tree = 0; tree < trees_each; tree++) - { - HostStorage storage = txn_spill_tests_tree(TXN_SPILL_TESTS_WRITERS[w].database, tree); - txn_spill_tests_based_t based; - - if (!txn_spill_tests_based(io, &storage, &based, &error)) - STD_RESULT_EXPECT_FAILED(2217, 39, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_spill_tests_based_drop(&based); - } - } - std_vec_new(&three.committed, sizeof(uint32), _Alignof(uint32)); + held = txn_memtable_testing_created(io, 9494); + host_shared_Held_drop(&held); run = simc_Run_new(1); - for (size_t w = 0; w < lengthof(TXN_SPILL_TESTS_WRITERS); w++) - { - SimcActor writer = {txn_spill_tests_writer, &writing[w]}; - - writing[w] = (txn_spill_tests_writing_t) - { - &three, TXN_SPILL_TESTS_WRITERS[w].database - }; - simc_Run_actor(&run, TXN_SPILL_TESTS_WRITERS[w].name, strlen(TXN_SPILL_TESTS_WRITERS[w].name), writer); - } - if (folding) - { - SimcActor folder = {txn_spill_tests_folder, &three}; - - simc_Run_actor(&run, "folder", 6, folder); - } + simc_Run_actor(&run, "owner", 5, owner); + simc_Run_actor(&run, "helper", 6, helper); + simc_Run_until(&run, "owner", 5, "doltgres-spill-case-written"); + simc_Run_until(&run, "helper", 6, "doltgres-spill-pended"); + simc_Run_script(&run, steps, lengthof(steps)); report = simc_Run_finish(run); - if (report.outcome.tag != SIMC_OUTCOME_ENDED) - { - size_t entries = 0; - size_t tuples = 0; - uint64 used; - - for (size_t w = 0; w < lengthof(TXN_SPILL_TESTS_WRITERS); w++) - for (uint32 tree = 0; tree < trees_each; tree++) - { - HostStorage storage = txn_spill_tests_tree(TXN_SPILL_TESTS_WRITERS[w].database, tree); - HostSharedHeld state; - StdVec files; - - if (!txn_memtable_testing_held(io, &storage, &state, &error)) - STD_RESULT_EXPECT_FAILED(2317, 49, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - files = txn_spill_tests_files_of(io, host_shared_Held_record(&state)); - for (size_t k = 0; k < files.len; k++) - { - if (STD_VEC_AT(&files, TxnSpillFile, k).kind != TXN_RUNS_KIND_SPILLED_TUPLES) - entries++; - if (STD_VEC_AT(&files, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_TUPLES) - tuples++; - } - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - host_shared_Held_drop(&state); - } - used = host_delta_used(); - { - StdFmtArg args[4] = {{&entries, std_usize_Display_fmt}, {&tuples, std_usize_Display_fmt}, - {&used, std_u64_Display_fmt}, {&report.outcome, simc_Outcome_Debug_fmt}}; - StdFmtArguments message = {unfinished, lengthof(unfinished), args}; - - std_panic_fmt(RUST_FILE, 2323, 13, &message); - } - } - all = true; - for (size_t k = 0; all && k < report.outcome.u.ended.len; k++) - all = simc_Ending_PartialEq_eq(&STD_VEC_AT(&report.outcome.u.ended, simc_TupleStringEnding_t, k).f1, - &returned); - if (!all) - { - StdFmtArg args[1] = {{&report.outcome, simc_Outcome_Debug_fmt}}; - StdFmtArguments message = {outcome, lengthof(outcome), args}; - - std_panic_fmt(RUST_FILE, 2329, 9, &message); - } - if (!folding) + txn_spill_tests_returned(&report, 2626, 67, lead, 2627); + if (!txn_memtable_testing_stated(io, 9494, &held, &error)) + STD_RESULT_EXPECT_FAILED(2628, 38, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + files = txn_spill_tests_files_of(io, host_shared_Held_record(&held)); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2631, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + count = files.len; + for (size_t k = 0; k < files.len; k++) { - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2333, 36, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - for (size_t w = 0; w < lengthof(TXN_SPILL_TESTS_WRITERS); w++) - for (uint32 tree = 0; tree < trees_each; tree++) - { - HostStorage storage = txn_spill_tests_tree(TXN_SPILL_TESTS_WRITERS[w].database, tree); - HostSharedHeld state; - uint64 numbers[2] = {0, rows - 1}; + const TxnSpillFile *file = &STD_VEC_AT(&files, TxnSpillFile, k); + bool seen = false; - if (!txn_memtable_testing_held(io, &storage, &state, &error)) - STD_RESULT_EXPECT_FAILED(2336, 45, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - for (size_t k = 0; k < lengthof(numbers); k++) - txn_spill_tests_one_version(2338, 21, 98, io, &hold, host_shared_Held_record(&state), numbers[k], - false); - host_shared_Held_drop(&state); - } - host_delta_Hold_drop(&hold); + for (size_t j = 0; !seen && j < k; j++) + seen = STD_VEC_AT(&files, TxnSpillFile, j).kind == file->kind && + STD_VEC_AT(&files, TxnSpillFile, j).number == file->number; + if (!seen) + distinct++; } - std_vec_free_Checked(&three.committed, sizeof(uint32), _Alignof(uint32)); + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + STD_ASSERT_EQ_MSG(2635, 9, distinct == count, &distinct, std_usize_Debug_fmt, &count, std_usize_Debug_fmt, + "no two files share a number"); + txn_spill_tests_one_version(2637, 13, 90, io, &hold, host_shared_Held_record(&held), 0, false); + txn_spill_tests_one_version(2637, 13, 90, io, &hold, host_shared_Held_record(&held), 49, false); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&held); simc_Report_drop(&report); simc_TestIo_drop(&test_io); } /* - * Bugs row 124's three writers: three transactions each write many small - * trees a row at a time, interleaved at every yield point by the run's seed, - * so their trees' small pages share the delta's blocks, and each writes past - * a third of the budget. A writer refused at it spills its trees one after - * another, and each part's removal frees pages whose blocks the others' pages - * keep, so no block frees; each part's entry takes the slot its own removal - * claimed. No writer commits until all three have written, so every version - * is a waiter's to spill, and every writer finishes with its rows read back. + * Bugs row 143's one file: a spill of more keys than one page of a filter + * holds writes one row file and one tuple-id file, each filter in several + * pages of its file, and every key reads back through the spill, its row's + * tuple id finding it through the tuple-id filter. */ void -test_txn_spill_tests_three_writers_of_small_trees_past_a_third_of_the_budget_each_spill_and_finish(void) +test_txn_spill_tests_a_spill_of_many_keys_writes_one_file_whose_filter_takes_pages(void) { - txn_spill_tests_three_writers(16, 40, false); + static const uint64 numbers[] = {0, 3000, 6000, 6999}; + static const TxnRunsKind wanted[] = {TXN_RUNS_KIND_SPILLED_TUPLES, TXN_RUNS_KIND_SPILLED_ROWS}; + SimcTestIo test_io; + HostIoIo *io; + HostSharedHeld held; + HostSharedRecord state; + HostStorage storage = txn_memtable_testing_storage(9483); + StdVec files; + HostDeltaHold hold; + HostError error; + uint64 xid; + bool spilled; + bool all = true; + + simc_size_delta(64 << 20); + test_io = simc_io(); + io = &test_io.io; + held = txn_memtable_testing_created(io, 9483); + state = host_shared_Held_record(&held); + xid = simc_begin().f0; + txn_spill_tests_tupled_rows(2656, 74, 2662, 51, io, state, &storage, xid, 0, 7000); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2664, 60, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2664, 9, spilled, "spilled(io, &storage(9483), Of::Rows, xid).unwrap()"); + files = txn_spill_tests_files_of(io, state); + txn_spill_tests_kinds_are(2667, &files, wanted, lengthof(wanted)); + for (size_t k = 0; all && k < files.len; k++) + all = STD_VEC_AT(&files, TxnSpillFile, k).bits > TXN_SPILL_PAGE_BITS; + STD_ASSERT(2668, 9, all, "files.iter().all(|file| file.bits > PAGE_BITS)"); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2669, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; k < lengthof(numbers); k++) + { + txn_spill_tests_one_version(2672, 13, 64, io, &hold, state, numbers[k], true); + txn_spill_tests_tuple_key_is(2673, io, &hold, state, numbers[k]); + } + host_delta_Hold_drop(&hold); + if (!txn_written_ended(io, &error)) + STD_RESULT_EXPECT_FAILED(2676, 28, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); } /* - * Bugs row 124's committed load: the three writers of twice the trees of half - * the rows each commit once written, so the budget the last one waits at is - * held by committed versions no waiter can spill; a fold of the committed - * storages, which an actor runs while a writer waits, as the server's fold - * trigger does, again while the writer still waits, since a fold refused by - * an older snapshot installs later, frees them, and every writer finishes. + * Bugs row 74's abort: an aborted transaction's spills leave its storages' + * chains with its take-back, so no reader reads its spilled versions, and + * another transaction's spill stays. */ void -test_txn_spill_tests_three_writers_waiting_on_committed_versions_finish_once_a_fold_installs(void) -{ - txn_spill_tests_three_writers(32, 20, true); -} - -/* txn_spill_tests_returned asserts at its lines that a run's actors all ended, panicking with its lead-in and the outcome when the run didn't end, and that each returned with no error. */ -static void -txn_spill_tests_returned(const SimcReport *report, uint32 line, uint32 col, const StdFmtPiece lead[2], - uint32 all_line) +test_txn_spill_tests_an_abort_takes_its_spills_out_of_the_chain(void) { - static const StdFmtPiece outcome[] = {STD_FMT_ARG(0)}; - SimcEnding returned = {.tag = SIMC_ENDING_RETURNED}; - StdFmtArg args[1] = {{&report->outcome, simc_Outcome_Debug_fmt}}; - bool all = true; - - if (report->outcome.tag != SIMC_OUTCOME_ENDED) + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9747); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9747); + uint64 mine = simc_begin().f0; + uint64 other = simc_begin().f0; + const struct { - StdFmtArguments message = {lead, 2, args}; + uint64 xid; + const char *key; + } pushes[] = {{other, "o"}, {mine, "m"}}; + HostXid aborted = {mine}; + StdVec files; + StdVec versions; + HostDeltaHold hold; + HostError error; + size_t one = 1; + bool spilled; + bool all = true; - std_panic_fmt(RUST_FILE, line, col, &message); - } - for (size_t k = 0; all && k < report->outcome.u.ended.len; k++) - all = simc_Ending_PartialEq_eq(&STD_VEC_AT(&report->outcome.u.ended, simc_TupleStringEnding_t, k).f1, - &returned); - if (!all) + for (size_t k = 0; k < lengthof(pushes); k++) { - StdFmtArguments message = {outcome, lengthof(outcome), args}; + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + uint64 touched; - std_panic_fmt(RUST_FILE, all_line, 9, &message); + if (!txn_written_touched(io, mine, &storage, TXN_WRITTEN_OF_ROWS, (const uint8 *) pushes[k].key, 1, 0, + &touched, &error)) + STD_RESULT_EXPECT_FAILED(2690, 74, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_memtable_table_push(io, state, (const uint8 *) pushes[k].key, 1, &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(2698, 50, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); } + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, other, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2700, 62, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2700, 9, spilled, "spilled(io, &storage(9747), Of::Rows, other).unwrap()"); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2701, 61, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2701, 9, spilled, "spilled(io, &storage(9747), Of::Rows, mine).unwrap()"); + simc_abort(aborted); + if (!txn_written_aborted(io, 0, &error)) + STD_RESULT_EXPECT_FAILED(2703, 30, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + files = txn_spill_tests_files_of(io, state); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2705, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; all && k < files.len; k++) + all = STD_VEC_AT(&files, TxnSpillFile, k).xid == other; + STD_ASSERT(2706, 9, all && files.len > 0, "left.iter().all(|&xid| xid == other) && !left.is_empty()"); + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (!txn_memtable_table_versions(io, &hold, state, (const uint8 *) "m", 1, &versions, &error)) + STD_RESULT_EXPECT_FAILED(2707, 57, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2707, 9, versions.len == 0, "table::versions(io, &hold, state, b\"m\").unwrap().is_empty()"); + txn_spill_tests_versions_free(&versions); + if (!txn_memtable_table_versions(io, &hold, state, (const uint8 *) "o", 1, &versions, &error)) + STD_RESULT_EXPECT_FAILED(2708, 60, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ(2708, 9, versions.len == 1, &versions.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt); + txn_spill_tests_versions_free(&versions); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); } -/* txn_spill_tests_fill takes the delta's pages of six kilobytes until it refuses one, as the spill cases' fillers do. */ +/* txn_spill_tests_u64s_Debug_fmt is a Vec's Debug. */ static bool -txn_spill_tests_fill(HostIoIo *io, StdVec *pages, HostError *error) +txn_spill_tests_u64s_Debug_fmt(const void *value, StdFormatter *f) { - HostDeltaPageSize size = {.bytes = 6 << 10, .words = 256}; - - for (;;) - { - StdVec taken; - HostDeltaFull full; - bool ok; + const StdVec *v = value; + StdDebugList list = std_fmt_debug_list(f); - if (!host_delta_alloc(io, &size, 1, &ok, &taken, &full, error)) - return false; - if (!ok) - return true; - std_vec_extend_from_slice_Checked(pages, taken.ptr, taken.len, sizeof(HostDeltaOwned), - _Alignof(HostDeltaOwned)); - std_vec_free_Checked(&taken, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); - } + for (size_t k = 0; k < v->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(v, uint64, k), std_u64_Debug_fmt); + return std_DebugList_finish(&list); } -/* txn_spill_tests_one_range_writer is the one-range case's writer: it writes its few rows, spills them at the filled budget, and ends its writes. */ -static bool -txn_spill_tests_one_range_writer(void *captures, HostIoIo *io, HostError *error) +/* txn_spill_tests_rows_xids returns the xids of a state's storage's files of spilled rows, newest first, as the redo case's spilled closure does. */ +static StdVec +txn_spill_tests_rows_xids(HostIoIo *io, HostSharedRecord state) { - HostStorage storage = txn_memtable_testing_storage(9488); - uint8 value[1000]; - HostSharedHeld held; - TxnMemtableOp op; - uint64 xid; - bool ok = true; - bool spilled; + StdVec files = txn_spill_tests_files_of(io, state); + StdVec xids; - if (!txn_memtable_testing_stated(io, 9488, &held, error)) - return false; - xid = simc_begin().f0; - memset(value, 7, sizeof(value)); - op = txn_memtable_testing_put(value, sizeof(value)); - for (uint64 number = 0; ok && number < 5; number++) + std_vec_new(&xids, sizeof(uint64), _Alignof(uint64)); + for (size_t k = 0; k < files.len; k++) { - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - size_t need = txn_memtable_version_encoded(&change.writer, change.op, change.shown, change.has_lsn, - change.lsn); - TxnMemtableVersionHandle handle; - uint64 touched; - uint8 key[8]; + const TxnSpillFile *file = &STD_VEC_AT(&files, TxnSpillFile, k); - std_u64_to_be_bytes(number, key); - ok = txn_written_touched(io, xid, &storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), need, &touched, error) && - txn_memtable_table_push(io, host_shared_Held_record(&held), key, sizeof(key), &change, &handle, error); + if (file->kind == TXN_RUNS_KIND_SPILLED_ROWS) + std_vec_push_Checked(&xids, &file->xid, sizeof(uint64), _Alignof(uint64)); } - ok = ok && host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-written", error) && - txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, error); - if (ok) - STD_ASSERT(2368, 13, spilled, "spilled(io, &storage(9488), Of::Rows, xid)?"); - ok = ok && host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-spilled", error) && - txn_written_ended(io, error); - txn_memtable_Op_drop(&op); - host_shared_Held_drop(&held); - return ok; + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + return xids; } -/* txn_spill_tests_filler is the one-range case's filler: it fills the delta, waits at a pause, and frees its pages. */ -static bool -txn_spill_tests_filler(void *captures, HostIoIo *io, HostError *error) +/* txn_spill_tests_xids_are asserts at its line that a Vec of xids is a list of them, and frees it. */ +static void +txn_spill_tests_xids_are(uint32 line, StdVec *xids, const uint64 *wanted, size_t len) { - StdVec pages; - bool ok; + StdVec wanted_vec = {(void *) wanted, len, len}; - std_vec_new(&pages, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); - ok = txn_spill_tests_fill(io, &pages, error) && - host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-filled", error); - std_vec_drop_Checked(&pages, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned), host_delta_Owned_drop); - return ok; + STD_ASSERT_EQ(line, 9, xids->len == len && memcmp(xids->ptr, wanted, len * sizeof(uint64)) == 0, xids, + txn_spill_tests_u64s_Debug_fmt, &wanted_vec, txn_spill_tests_u64s_Debug_fmt); + std_vec_free_Checked(xids, sizeof(uint64), _Alignof(uint64)); } /* - * Bugs row 124's one-range tree: a tree of a few rows, its one range the - * memtable's floor, spills at a delta another process filled; its manifest - * takes none of the delta (bugs row 143), so the spill lists it at once. + * Bugs row 74's redo: the startup process, which has no transaction of its + * own, spills the replayed transactions' trees when it finds the delta full, + * the largest first and whatever their size, as a backend spills its own + * (bugs row 124), until none is left. */ void -test_txn_spill_tests_a_one_range_tree_spills_at_a_full_budget(void) -{ - static const StdFmtPiece lead[] = {STD_FMT_LIT("the spill didn't link: "), STD_FMT_ARG(0)}; - SimcStep steps[2] = {{"filler", 6, PG_UINT64_MAX}, {"writer", 6, PG_UINT64_MAX}}; - SimcActor writer = {txn_spill_tests_one_range_writer}; - SimcActor filler = {txn_spill_tests_filler}; - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; - SimcRun run; - SimcReport report; - HostDeltaHold hold; - HostError error; - - simc_size_delta(2 << 20); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9488); - host_shared_Held_drop(&held); - run = simc_Run_new(1); - simc_Run_actor(&run, "writer", 6, writer); - simc_Run_actor(&run, "filler", 6, filler); - simc_Run_until(&run, "writer", 6, "doltgres-spill-case-written"); - simc_Run_until(&run, "filler", 6, "doltgres-spill-case-filled"); - simc_Run_until(&run, "writer", 6, "doltgres-spill-case-spilled"); - simc_Run_script(&run, steps, lengthof(steps)); - report = simc_Run_finish(run); - txn_spill_tests_returned(&report, 2387, 13, lead, 2389); - if (!txn_memtable_testing_stated(io, 9488, &held, &error)) - STD_RESULT_EXPECT_FAILED(2390, 37, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2391, 36, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - for (uint64 number = 0; number < 5; number++) - txn_spill_tests_one_version(2393, 13, 89, io, &hold, host_shared_Held_record(&held), number, false); - host_delta_Hold_drop(&hold); - host_shared_Held_drop(&held); - simc_Report_drop(&report); - simc_TestIo_drop(&test_io); -} - -/* txn_spill_tests_tupled_rows writes a transaction's rows of a range of numbers to a state's memtable, each counted in its written trees with no size and each a put whose tuple id is its number plus one. */ -static void -txn_spill_tests_tupled_rows(uint32 touch_line, uint32 touch_col, uint32 push_line, uint32 push_col, HostIoIo *io, - HostSharedRecord state, const HostStorage *storage, uint64 xid, uint64 from, uint64 to) +test_txn_spill_tests_redo_over_budget_spills_the_largest_replayed_transaction(void) { + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9749); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9749); + uint64 small = simc_begin().f0; + uint64 large = simc_begin().f0; + const struct + { + uint64 xid; + const char *key; + } pushes[] = {{small, "s"}, {large, "l"}}; + const uint64 first[] = {large}; + const uint64 both[] = {small, large}; + StdVec xids; HostError error; + uint64 count; + bool relieved; - for (uint64 number = from; number < to; number++) + for (size_t k = 0; k < lengthof(pushes); k++) { TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}, + .has_lsn = true, .lsn = 9}; TxnMemtableVersionHandle handle; - uint64 touched; - uint8 key[8]; - std_u64_to_be_bytes(number, key); - if (!txn_written_touched(io, xid, storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), 0, &touched, &error)) - STD_RESULT_EXPECT_FAILED(touch_line, touch_col, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - op.u.put.tuple.f0 = number + 1; - if (!txn_memtable_table_push(io, state, key, sizeof(key), &change, &handle, &error)) - STD_RESULT_EXPECT_FAILED(push_line, push_col, "called `Result::unwrap()` on an `Err` value", &error, + if (!txn_memtable_table_push(io, state, (const uint8 *) pushes[k].key, 1, &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(2730, 50, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); txn_memtable_Op_drop(&op); } + if (!txn_spill_replayed(io, &storage, TXN_WRITTEN_OF_ROWS, small, 10, &count, &error)) + STD_RESULT_EXPECT_FAILED(2732, 59, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_spill_replayed(io, &storage, TXN_WRITTEN_OF_ROWS, large, (size_t) TXN_SPILL_LOW, &count, &error)) + STD_RESULT_EXPECT_FAILED(2733, 69, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_spill_relieved(io, &relieved, &error)) + STD_RESULT_EXPECT_FAILED(2738, 30, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2738, 9, relieved, "relieved(io).unwrap()"); + xids = txn_spill_tests_rows_xids(io, state); + txn_spill_tests_xids_are(2739, &xids, first, lengthof(first)); + if (!txn_spill_relieved(io, &relieved, &error)) + STD_RESULT_EXPECT_FAILED(2740, 30, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2740, 9, relieved, "relieved(io).unwrap()"); + xids = txn_spill_tests_rows_xids(io, state); + txn_spill_tests_xids_are(2741, &xids, both, lengthof(both)); + if (!txn_spill_relieved(io, &relieved, &error)) + STD_RESULT_EXPECT_FAILED(2742, 31, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2742, 9, !relieved, "!relieved(io).unwrap()"); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); } -/* txn_spill_tests_kind_number_t is a spill file's kind and number, as the merge case lists them. */ -typedef struct txn_spill_tests_kind_number_t -{ - TxnRunsKind kind; - uint32 number; -} txn_spill_tests_kind_number_t; - -/* txn_spill_tests_kind_numbers_Debug_fmt is a Vec<(Kind, u32)>'s Debug. */ +/* txn_spill_tests_u32s_Debug_fmt is a Vec's Debug. */ static bool -txn_spill_tests_kind_numbers_Debug_fmt(const void *value, StdFormatter *f) +txn_spill_tests_u32s_Debug_fmt(const void *value, StdFormatter *f) { - const StdVec *kinds = value; + const StdVec *v = value; StdDebugList list = std_fmt_debug_list(f); - for (size_t k = 0; k < kinds->len; k++) - { - const txn_spill_tests_kind_number_t *kind = &STD_VEC_AT(kinds, txn_spill_tests_kind_number_t, k); - StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&kind->kind, txn_spill_tests_Kind_Debug_fmt), - STD_ITEM(&kind->number, std_u32_Debug_fmt)); - - std_DebugList_entry(&list, &tuple, std_tuple_Debug_fmt); - } + for (size_t k = 0; k < v->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(v, uint32, k), std_u32_Debug_fmt); return std_DebugList_finish(&list); } -/* txn_spill_tests_tuple_key_is asserts at its line that a number's row's tuple id, its number plus one, finds the number's key, the message the number. */ +/* txn_spill_tests_put_by pushes a transaction's put of a value to a key of a state's memtable, as the tiers case's push closure does. */ static void -txn_spill_tests_tuple_key_is(uint32 line, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, - uint64 number) +txn_spill_tests_put_by(HostIoIo *io, HostSharedRecord state, uint64 xid, const char *key, const char *value) { - static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; - HostRow tid = {number + 1}; - TxnMemtableMapEntry found; - TxnMemtableMapEntry key = {.has_value = true}; + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) value, strlen(value)); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; HostError error; - uint8 be[8]; - if (!txn_spill_tuple_key(io, hold, state, tid, &found.has_value, &found.value, &error)) - STD_RESULT_EXPECT_FAILED(line, 69, "called `Result::unwrap()` on an `Err` value", &error, + if (!txn_memtable_table_push(io, state, (const uint8 *) key, strlen(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(2764, 50, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - std_u64_to_be_bytes(number, be); - key.value = (StdVec) - { - be, sizeof(be), sizeof(be) - }; - if (!txn_memtable_map_tests_seen_eq(&found, &key)) - { - StdFmtArg args[1] = {{&number, std_u64_Display_fmt}}; - StdFmtArguments message = {pieces, lengthof(pieces), args}; - - std_assert_failed(RUST_FILE, line, 13, "==", &found, txn_memtable_map_tests_seen_Debug_fmt, &key, - txn_memtable_map_tests_seen_Debug_fmt, &message); - } - txn_memtable_map_tests_seen_drop(&found); + txn_memtable_Op_drop(&op); } -/* - * Bugs row 124's merge of tuple ids: a transaction's two spills of one tree, - * the second as large as the first, merge into one row file, and the tuple-id - * files their entries name merge into one of the same number, which the - * merged entry names; every row's tuple id still finds its key. - */ -void -test_txn_spill_tests_a_merge_of_row_spills_merges_the_tuple_ids_their_entries_name(void) +/* txn_spill_tests_values_are asserts at its line that a key's versions' values in a state's memtable and spills are a list, newest first, as the tiers case's values closure reads them. */ +static void +txn_spill_tests_values_are(uint32 line, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, + const char *key, const char *const *wanted, size_t len) { - static const uint64 numbers[] = {0, 99, 100, 199}; - txn_spill_tests_kind_number_t wanted[] = {{TXN_RUNS_KIND_SPILLED_TUPLES, 2}, {TXN_RUNS_KIND_SPILLED_ROWS, 2}}; - StdVec wanted_vec = {wanted, lengthof(wanted), lengthof(wanted)}; - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; - HostSharedRecord state; - HostStorage storage = txn_memtable_testing_storage(9489); - StdVec files; - StdVec kinds; - HostDeltaHold hold; + StdVec versions; + StdVec values; + StdVec expected; HostError error; - uint64 xid; - bool spilled; - bool eq; - simc_size_delta(64 << 20); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9489); - state = host_shared_Held_record(&held); - xid = simc_begin().f0; - for (uint64 half = 0; half < 2; half++) - { - txn_spill_tests_tupled_rows(2412, 78, 2418, 55, io, state, &storage, xid, half * 100, half * 100 + 100); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2420, 64, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2420, 13, spilled, "spilled(io, &storage(9489), Of::Rows, xid).unwrap()"); - } - files = txn_spill_tests_files_of(io, state); - std_vec_with_capacity_Checked(&kinds, files.len, sizeof(txn_spill_tests_kind_number_t), - _Alignof(txn_spill_tests_kind_number_t)); - for (size_t k = 0; k < files.len; k++) + if (!txn_memtable_table_versions(io, hold, state, (const uint8 *) key, strlen(key), &versions, &error)) + STD_RESULT_EXPECT_FAILED(2778, 18, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_with_capacity_Checked(&values, versions.len, sizeof(StdVec), _Alignof(StdVec)); + for (size_t k = 0; k < versions.len; k++) { - const TxnSpillFile *file = &STD_VEC_AT(&files, TxnSpillFile, k); - txn_spill_tests_kind_number_t kind = {file->kind, file->number}; + TxnMemtableVersionRecord record = + txn_memtable_version_Version_record(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, k)); + StdVec value = txn_spill_tests_value(&record); - std_vec_push_Checked(&kinds, &kind, sizeof(kind), _Alignof(txn_spill_tests_kind_number_t)); + std_vec_push_Checked(&values, &value, sizeof(StdVec), _Alignof(StdVec)); } - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2424, 36, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - eq = kinds.len == lengthof(wanted); - for (size_t k = 0; eq && k < lengthof(wanted); k++) - eq = STD_VEC_AT(&kinds, txn_spill_tests_kind_number_t, k).kind == wanted[k].kind && - STD_VEC_AT(&kinds, txn_spill_tests_kind_number_t, k).number == wanted[k].number; - STD_ASSERT_EQ(2425, 9, eq, &kinds, txn_spill_tests_kind_numbers_Debug_fmt, &wanted_vec, - txn_spill_tests_kind_numbers_Debug_fmt); - for (size_t k = 0; k < lengthof(numbers); k++) - txn_spill_tests_tuple_key_is(2428, io, &hold, state, numbers[k]); - host_delta_Hold_drop(&hold); - if (!txn_written_ended(io, &error)) - STD_RESULT_EXPECT_FAILED(2431, 28, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - std_vec_free_Checked(&kinds, sizeof(txn_spill_tests_kind_number_t), _Alignof(txn_spill_tests_kind_number_t)); - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - host_shared_Held_drop(&held); - simc_TestIo_drop(&test_io); + txn_spill_tests_versions_free(&versions); + txn_memtable_table_tests_keys_of(wanted, len, &expected); + STD_ASSERT_EQ(line, 9, txn_memtable_table_tests_keys_eq(&values, &expected), &values, + txn_memtable_table_tests_keys_Debug_fmt, &expected, txn_memtable_table_tests_keys_Debug_fmt); + txn_memtable_table_tests_keys_free(&expected); + txn_memtable_table_tests_keys_free(&values); } /* - * Bugs row 143's merge at a full budget: a transaction's two small spills of - * one tree at a delta another process filled merge into one file, listed in - * place of the two with no page of the delta, and every key reads back - * through it. + * Bugs row 74's tiers: a transaction's second spill of a storage's rows, no + * larger than its first, merges with it into one file, a key written again + * after the first spill holding its newer version first, and the merged + * files leave the chain. */ void -test_txn_spill_tests_a_merge_at_a_full_budget_lists_its_file_in_place_of_its_sources(void) +test_txn_spill_tests_a_second_spill_merges_with_the_first_keeping_its_versions_order(void) { - static const uint64 numbers[] = {0, 19}; - static const TxnRunsKind wanted[] = {TXN_RUNS_KIND_SPILLED_TUPLES, TXN_RUNS_KIND_SPILLED_ROWS}; - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; - HostSharedRecord state; - HostStorage storage = txn_memtable_testing_storage(9490); - StdVec filler; - HostDeltaHold hold; + static const char *const a[] = {"1"}; + static const char *const b[] = {"3", "2"}; + static const char *const c[] = {"4"}; + static const uint32 merged[] = {2}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9516); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9516); + uint64 mine = simc_begin().f0; StdVec files; + StdVec rows; + StdVec merged_vec = {(void *) merged, lengthof(merged), lengthof(merged)}; + HostDeltaHold hold; HostError error; - uint64 xid; bool spilled; - simc_size_delta(2 << 20); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9490); - state = host_shared_Held_record(&held); - xid = simc_begin().f0; - std_vec_new(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); - for (uint64 half = 0; half < 2; half++) - { - txn_spill_tests_tupled_rows(2450, 78, 2456, 55, io, state, &storage, xid, half * 10, half * 10 + 10); - if ((filler.len > 0 || half * 10 > 0) && !txn_spill_tests_fill(io, &filler, &error)) - STD_RESULT_EXPECT_FAILED(2459, 96, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2463, 64, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2463, 13, spilled, "spilled(io, &storage(9490), Of::Rows, xid).unwrap()"); - } + txn_spill_tests_put_by(io, state, mine, "a", "1"); + txn_spill_tests_put_by(io, state, mine, "b", "2"); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2768, 61, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2768, 9, spilled, "spilled(io, &storage(9516), Of::Rows, mine).unwrap()"); + txn_spill_tests_put_by(io, state, mine, "b", "3"); + txn_spill_tests_put_by(io, state, mine, "c", "4"); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2771, 61, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2771, 9, spilled, "spilled(io, &storage(9516), Of::Rows, mine).unwrap()"); files = txn_spill_tests_files_of(io, state); - txn_spill_tests_kinds_are(2466, &files, wanted, lengthof(wanted)); + std_vec_new(&rows, sizeof(uint32), _Alignof(uint32)); + for (size_t k = 0; k < files.len; k++) + { + const TxnSpillFile *file = &STD_VEC_AT(&files, TxnSpillFile, k); + + if (file->kind == TXN_RUNS_KIND_SPILLED_ROWS) + std_vec_push_Checked(&rows, &file->number, sizeof(uint32), _Alignof(uint32)); + } std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2467, 36, "called `Result::unwrap()` on an `Err` value", &error, + STD_RESULT_EXPECT_FAILED(2774, 36, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - for (size_t k = 0; k < lengthof(numbers); k++) - { - txn_spill_tests_one_version(2470, 13, 64, io, &hold, state, numbers[k], true); - txn_spill_tests_tuple_key_is(2471, io, &hold, state, numbers[k]); - } + STD_ASSERT_EQ(2775, 9, rows.len == 1 && STD_VEC_AT(&rows, uint32, 0) == 2, &rows, txn_spill_tests_u32s_Debug_fmt, + &merged_vec, txn_spill_tests_u32s_Debug_fmt); + std_vec_free_Checked(&rows, sizeof(uint32), _Alignof(uint32)); + txn_spill_tests_values_are(2786, io, &hold, state, "a", a, lengthof(a)); + txn_spill_tests_values_are(2787, io, &hold, state, "b", b, lengthof(b)); + txn_spill_tests_values_are(2788, io, &hold, state, "c", c, lengthof(c)); host_delta_Hold_drop(&hold); - std_vec_drop_Checked(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned), host_delta_Owned_drop); - if (!txn_written_ended(io, &error)) - STD_RESULT_EXPECT_FAILED(2474, 28, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); } -/* txn_spill_tests_merger is the merge window case's merger: its transaction's second spill, which merges with its first. */ -static bool -txn_spill_tests_merger(void *captures, HostIoIo *io, HostError *error) +/* + * Bugs row 107's threshold (STORAGE_DESIGN §III.3): a spill's threshold is an + * eighth of the budget, and at the least delta_memory that is below LOW, so a + * recovery's replayed transactions there spill before the delta's fixed + * records leave no room for any of them to reach LOW. + */ +void +test_txn_spill_tests_a_small_budgets_threshold_is_an_eighth_of_it(void) { - HostStorage storage = txn_memtable_testing_storage(9491); - bool spilled; + uint64 threshold; + uint64 eighth = ((uint64) 1 << 20) / TXN_SPILL_SHARE; - return txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, *(const uint64 *) captures, &spilled, error); + simc_size_delta(1 << 20); + threshold = txn_spill_threshold(); + STD_ASSERT_EQ(2798, 9, threshold == eighth, &threshold, std_u64_Debug_fmt, &eighth, std_u64_Debug_fmt); + STD_ASSERT(2799, 9, txn_spill_threshold() < TXN_SPILL_LOW, "threshold() < LOW"); } -/* txn_spill_tests_found_t is the merge window case's reader's finding: whether it read, and its count of the first key's versions. */ -typedef struct txn_spill_tests_found_t -{ - bool some; - size_t count; -} txn_spill_tests_found_t; - -/* txn_spill_tests_found_Debug_fmt is the reader's finding's Debug, an Option. */ -static bool -txn_spill_tests_found_Debug_fmt(const void *value, StdFormatter *f) +/* txn_spill_tests_pushed_rows pushes a transaction's version of each of a range of numbered keys to a storage's rows, counted toward its spill as a writer's are. */ +static void +txn_spill_tests_pushed_rows(HostIoIo *io, HostSharedRecord state, const HostStorage *at, uint64 xid, uint64 start, + uint64 end) { - const txn_spill_tests_found_t *found = value; - StdOptionRef option = {found->some, STD_ITEM(&found->count, std_usize_Debug_fmt)}; + for (uint64 number = start; number < end; number++) + { + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + HostError error; + uint64 touched; + uint8 key[8]; - return std_option_Debug_fmt(&option, f); + std_u64_to_be_bytes(number, key); + if (!txn_written_touched(io, xid, at, TXN_WRITTEN_OF_ROWS, key, sizeof(key), 0, &touched, &error)) + STD_RESULT_EXPECT_FAILED(2814, 62, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_memtable_table_push(io, state, key, sizeof(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(2815, 51, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + } } -/* txn_spill_tests_window_reader is the merge window case's reader: it counts the first key's versions. */ -static bool -txn_spill_tests_window_reader(void *captures, HostIoIo *io, HostError *error) +/* txn_spill_tests_relation returns the storage of a relation of a test's database, numbered past the first. */ +static HostStorage +txn_spill_tests_relation(uint32 database, uint32 number) { - txn_spill_tests_found_t *found = captures; - HostSharedHeld held; - HostDeltaHold hold; - StdVec versions; - uint8 key[8]; - bool ok; + HostStorage storage = txn_memtable_testing_storage(database); - if (!txn_memtable_testing_stated(io, 9491, &held, error)) - return false; - if (!host_delta_hold(io, &hold, error)) - { - host_shared_Held_drop(&held); - return false; - } - std_u64_to_be_bytes(0, key); - ok = txn_memtable_table_versions(io, &hold, host_shared_Held_record(&held), key, sizeof(key), &versions, error); - if (ok) - { - *found = (txn_spill_tests_found_t) - { - true, versions.len - }; - txn_spill_tests_versions_free(&versions); - } - host_delta_Hold_drop(&hold); - host_shared_Held_drop(&held); - return ok; + storage.relation = 16384 + number; + return storage; } /* - * Bugs row 124's merge window: a reader that read the slots before a merge - * pended its file, and lists the chain after the merge unchained its sources - * but before it linked the merged file, sees the chain changed, starts over, - * and reads the key's version once, through the pended merged file. + * Bugs row 143's many storages: one transaction spills more storages than a + * 256 KB delta holds in 1 KB slots, its spills' bookkeeping taking none of + * the delta, so the delta's used bytes after each later spill are no more + * than after its first, and every storage's rows read back through its + * spill. */ void -test_txn_spill_tests_a_reader_across_a_merges_unchain_reads_each_version_once(void) +test_txn_spill_tests_a_transaction_spilling_more_storages_than_the_budget_holds_pages_completes(void) { - static const StdFmtPiece lead[] = {STD_FMT_LIT("the run didn't end: "), STD_FMT_ARG(0)}; - SimcStep steps[2] = {{"reader", 6, PG_UINT64_MAX}, {"merger", 6, PG_UINT64_MAX}}; - txn_spill_tests_found_t found = {0}; - txn_spill_tests_found_t once = {true, 1}; + static const StdFmtPiece past[] = {STD_FMT_LIT("after "), STD_FMT_ARG(0), STD_FMT_LIT(" the delta uses "), + STD_FMT_ARG(1), STD_FMT_LIT(" past "), STD_FMT_ARG(2)}; + static const uint32 read[] = {0, 299}; SimcTestIo test_io; HostIoIo *io; HostSharedHeld held; - HostStorage storage = txn_memtable_testing_storage(9491); - SimcActor merger = {txn_spill_tests_merger}; - SimcActor reader = {txn_spill_tests_window_reader, &found}; - SimcRun run; - SimcReport report; HostError error; uint64 xid; - bool spilled; + uint64 first = 0; + bool has_first = false; - simc_size_delta(64 << 20); + simc_size_delta(256 << 10); test_io = simc_io(); io = &test_io.io; - held = txn_memtable_testing_created(io, 9491); + held = txn_memtable_testing_created(io, 9510); + host_shared_Held_drop(&held); xid = simc_begin().f0; - for (uint64 half = 0; half < 2; half++) + for (uint32 number = 0; number < 300; number++) { - for (uint64 number = half * 10; number < half * 10 + 10; number++) - { - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - TxnMemtableVersionHandle handle; - uint64 touched; - uint8 key[8]; + HostStorage at = txn_spill_tests_relation(9510, number); + HostSharedHeld state; + bool spilled; + uint64 used; - std_u64_to_be_bytes(number, key); - if (!txn_written_touched(io, xid, &storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), 0, &touched, &error)) - STD_RESULT_EXPECT_FAILED(2491, 78, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_memtable_table_push(io, host_shared_Held_record(&held), key, sizeof(key), &change, &handle, - &error)) - STD_RESULT_EXPECT_FAILED(2499, 63, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_memtable_Op_drop(&op); + if (!txn_memtable_testing_held(io, &at, &state, &error)) + STD_RESULT_EXPECT_FAILED(2837, 59, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), &at, xid, 0, 4); + if (!txn_spill_spilled(io, &at, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2839, 73, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2839, 13, spilled, "spilled(io, &relation(9510, number), Of::Rows, xid).unwrap()"); + if (!has_first) + { + first = host_delta_used(); + has_first = true; } - if (half == 0) + used = first; + if (!(host_delta_used() <= used)) { - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2502, 68, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2502, 17, spilled, "spilled(io, &storage(9491), Of::Rows, xid).unwrap()"); + uint64 now = host_delta_used(); + StdFmtArg args[3] = {{&number, std_u32_Display_fmt}, {&now, std_u64_Display_fmt}, + {&used, std_u64_Display_fmt}}; + StdFmtArguments message = {past, lengthof(past), args}; + + std_panic_fmt(RUST_FILE, 2841, 13, &message); } + host_shared_Held_drop(&state); + } + for (size_t k = 0; k < lengthof(read); k++) + { + HostStorage at = txn_spill_tests_relation(9510, read[k]); + HostSharedHeld state; + HostDeltaHold hold; + + if (!txn_memtable_testing_held(io, &at, &state, &error)) + STD_RESULT_EXPECT_FAILED(2844, 59, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2845, 40, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_spill_tests_one_version(2846, 13, 88, io, &hold, host_shared_Held_record(&state), 3, false); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&state); } - host_shared_Held_drop(&held); - merger.captures = &xid; - run = simc_Run_new(1); - simc_Run_actor(&run, "merger", 6, merger); - simc_Run_actor(&run, "reader", 6, reader); - simc_Run_until(&run, "merger", 6, "doltgres-spill-linked"); - simc_Run_until(&run, "reader", 6, "doltgres-spill-slots-read"); - simc_Run_until(&run, "merger", 6, "doltgres-spill-merge-unchained"); - simc_Run_script(&run, steps, lengthof(steps)); - report = simc_Run_finish(run); - txn_spill_tests_returned(&report, 2521, 67, lead, 2522); - STD_ASSERT_EQ(2523, 9, found.some && found.count == 1, &found, txn_spill_tests_found_Debug_fmt, &once, - txn_spill_tests_found_Debug_fmt); if (!txn_written_ended(io, &error)) - STD_RESULT_EXPECT_FAILED(2524, 28, "called `Result::unwrap()` on an `Err` value", &error, + STD_RESULT_EXPECT_FAILED(2848, 28, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - simc_Report_drop(&report); simc_TestIo_drop(&test_io); } -/* txn_spill_tests_paused_wrote writes a transaction's rows of a count to each of a database's nine trees, a tree at a time, as the paused holder case's wrote closure does. */ -static bool -txn_spill_tests_paused_wrote(HostIoIo *io, uint32 database, uint64 rows, HostError *error) +/* + * Bugs row 143's two at once: two transactions spill the same storages + * crosswise, one first in a storage and the other first in the next, more + * storages than a 256 KB delta holds in 1 KB slots, both completing, and + * once both abort, every manifest lists nothing and the delta's used bytes + * are no more than after the first storage's spills. + */ +void +test_txn_spill_tests_two_transactions_spilling_the_same_storages_crosswise_complete(void) { - uint64 xid = simc_begin().f0; - uint8 value[1000]; - TxnMemtableOp op; - bool ok = true; + static const StdFmtPiece past[] = {STD_FMT_LIT("the delta uses "), STD_FMT_ARG(0), STD_FMT_LIT(" past "), + STD_FMT_ARG(1)}; + SimcTestIo test_io; + HostIoIo *io; + HostSharedHeld held; + HostStorage storages[300]; + HostError error; + uint64 one; + uint64 two; + uint64 first = 0; + bool has_first = false; - memset(value, 7, sizeof(value)); - op = txn_memtable_testing_put(value, sizeof(value)); - for (uint32 tree = 0; ok && tree < 9; tree++) + simc_size_delta(256 << 10); + test_io = simc_io(); + io = &test_io.io; + held = txn_memtable_testing_created(io, 9511); + host_shared_Held_drop(&held); + one = simc_begin().f0; + two = simc_begin().f0; + for (uint32 number = 0; number < lengthof(storages); number++) + storages[number] = txn_spill_tests_relation(9511, number); + for (size_t number = 0; number < lengthof(storages); number++) { - HostStorage storage = txn_spill_tests_tree(database, tree); + const HostStorage *at = &storages[number]; + uint64 order[2] = {number % 2 == 0 ? one : two, number % 2 == 0 ? two : one}; HostSharedHeld state; - if (!txn_memtable_testing_held(io, &storage, &state, error)) + if (!txn_memtable_testing_held(io, at, &state, &error)) + STD_RESULT_EXPECT_FAILED(2866, 38, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), at, one, 0, 2); + txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), at, two, 2, 4); + for (size_t k = 0; k < lengthof(order); k++) { - ok = false; - break; + bool spilled; + + if (!txn_spill_spilled(io, at, TXN_WRITTEN_OF_ROWS, order[k], &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2870, 56, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(2870, 17, spilled, "spilled(io, at, Of::Rows, xid).unwrap()"); } - for (uint64 number = 0; ok && number < rows; number++) + if (!has_first) { - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - size_t need = txn_memtable_version_encoded(&change.writer, change.op, change.shown, - change.has_lsn, change.lsn); - TxnMemtableVersionHandle handle; - uint64 touched; - uint8 key[8]; - - std_u64_to_be_bytes(number, key); - ok = txn_written_touched(io, xid, &storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), need, &touched, - error) && - txn_memtable_table_push(io, host_shared_Held_record(&state), key, sizeof(key), &change, &handle, - error); + first = host_delta_used(); + has_first = true; } host_shared_Held_drop(&state); } - txn_memtable_Op_drop(&op); - return ok; -} - -/* txn_spill_tests_holder is the paused holder case's holder: it writes its trees, waits at a pause, and ends its writes. */ -static bool -txn_spill_tests_holder(void *captures, HostIoIo *io, HostError *error) -{ - return txn_spill_tests_paused_wrote(io, 9492, 190, error) && - host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-held", error) && - txn_written_ended(io, error); -} + if (!txn_spill_aborted(io, one, storages, lengthof(storages), 0, &error)) + STD_RESULT_EXPECT_FAILED(2875, 41, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_spill_aborted(io, two, storages, lengthof(storages), 0, &error)) + STD_RESULT_EXPECT_FAILED(2875, 41, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; k < 2; k++) + { + const HostStorage *at = &storages[k == 0 ? 0 : 299]; + HostSharedHeld state; + StdVec files; -/* txn_spill_tests_relieving_writer is the paused holder case's writer: its relief spills the paused transaction's versions. */ -static bool -txn_spill_tests_relieving_writer(void *captures, HostIoIo *io, HostError *error) -{ - bool relieved; + if (!txn_memtable_testing_held(io, at, &state, &error)) + STD_RESULT_EXPECT_FAILED(2878, 38, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + files = txn_spill_tests_files_of(io, host_shared_Held_record(&state)); + STD_ASSERT(2879, 13, files.len == 0, "files_of(io, state.record()).is_empty()"); + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + host_shared_Held_drop(&state); + } + if (!(host_delta_used() <= first)) + { + uint64 now = host_delta_used(); + StdFmtArg args[2] = {{&now, std_u64_Display_fmt}, {&first, std_u64_Display_fmt}}; + StdFmtArguments message = {past, lengthof(past), args}; - if (!txn_spill_relieved(io, &relieved, error)) - return false; - STD_ASSERT_MSG(2570, 13, relieved, "the paused transaction's versions spill"); - return true; + std_panic_fmt(RUST_FILE, 2882, 9, &message); + } + if (!txn_written_ended(io, &error)) + STD_RESULT_EXPECT_FAILED(2883, 28, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_TestIo_drop(&test_io); } /* - * Bugs row 144's paused holder: a transaction that wrote trees each under the - * spill threshold waits at a pause before its commit, not at the budget, so - * it spills nothing itself; a process with nothing of its own to spill - * relieves the budget by spilling the paused transaction's versions from - * every tree, and the paused one's rows read back after its commit. + * Bugs row 143's many spillers: twenty transactions, more than any fixed + * count of a storage's spillers, each spill one storage at a delta another + * process filled, every one completing with no page of the delta, and every + * transaction's rows read back. */ void -test_txn_spill_tests_a_process_with_nothing_of_its_own_spills_a_paused_transactions_versions(void) +test_txn_spill_tests_more_transactions_than_any_fixed_count_spill_one_storage_at_a_full_budget(void) { - static const StdFmtPiece lead[] = {STD_FMT_LIT("the writer didn't finish: "), STD_FMT_ARG(0)}; - SimcStep steps[2] = {{"writer", 6, PG_UINT64_MAX}, {"holder", 6, PG_UINT64_MAX}}; - SimcActor holder = {txn_spill_tests_holder}; - SimcActor writer = {txn_spill_tests_relieving_writer}; + static const uint64 numbers[] = {0, 99, 199}; SimcTestIo test_io; HostIoIo *io; HostSharedHeld held; - SimcRun run; - SimcReport report; + HostSharedRecord state; + HostStorage storage = txn_memtable_testing_storage(9512); + uint64 xids[20]; + StdVec filler; + StdVec files; HostDeltaHold hold; HostError error; + size_t rows = 0; + size_t count = lengthof(xids); simc_size_delta(2 << 20); test_io = simc_io(); io = &test_io.io; - held = txn_memtable_testing_created(io, 9492); - host_shared_Held_drop(&held); - run = simc_Run_new(1); - simc_Run_actor(&run, "holder", 6, holder); - simc_Run_actor(&run, "writer", 6, writer); - simc_Run_until(&run, "holder", 6, "doltgres-spill-case-held"); - simc_Run_script(&run, steps, lengthof(steps)); - report = simc_Run_finish(run); - txn_spill_tests_returned(&report, 2577, 13, lead, 2579); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2580, 36, "called `Result::unwrap()` on an `Err` value", &error, + held = txn_memtable_testing_created(io, 9512); + state = host_shared_Held_record(&held); + for (size_t k = 0; k < lengthof(xids); k++) + xids[k] = simc_begin().f0; + for (uint64 at = 0; at < lengthof(xids); at++) + txn_spill_tests_pushed_rows(io, state, &storage, xids[at], at * 10, at * 10 + 10); + std_vec_new(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); + if (!txn_spill_tests_fill(io, &filler, &error)) + STD_RESULT_EXPECT_FAILED(2902, 88, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - for (uint32 tree = 0; tree < 9; tree++) + for (size_t k = 0; k < lengthof(xids); k++) { - HostStorage storage = txn_spill_tests_tree(9492, tree); - HostSharedHeld state; - StdVec files; + bool spilled; - if (!txn_memtable_testing_held(io, &storage, &state, &error)) - STD_RESULT_EXPECT_FAILED(2582, 41, "called `Result::unwrap()` on an `Err` value", &error, + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xids[k], &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2906, 64, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - files = txn_spill_tests_files_of(io, host_shared_Held_record(&state)); - STD_ASSERT_MSG(2583, 13, files.len > 0, "every tree spilled"); - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - txn_spill_tests_one_version(2585, 17, 94, io, &hold, host_shared_Held_record(&state), 0, false); - txn_spill_tests_one_version(2585, 17, 94, io, &hold, host_shared_Held_record(&state), 189, false); - host_shared_Held_drop(&state); + STD_ASSERT(2906, 13, spilled, "spilled(io, &storage(9512), Of::Rows, xid).unwrap()"); } + files = txn_spill_tests_files_of(io, state); + for (size_t k = 0; k < files.len; k++) + if (STD_VEC_AT(&files, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_ROWS) + rows++; + std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + STD_ASSERT_EQ(2909, 9, rows == count, &rows, std_usize_Debug_fmt, &count, std_usize_Debug_fmt); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2910, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; k < lengthof(numbers); k++) + txn_spill_tests_one_version(2912, 13, 81, io, &hold, state, numbers[k], true); host_delta_Hold_drop(&hold); - simc_Report_drop(&report); + std_vec_drop_Checked(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned), host_delta_Owned_drop); + if (!txn_written_ended(io, &error)) + STD_RESULT_EXPECT_FAILED(2915, 28, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); } -/* txn_spill_tests_owner is the turn case's owner: it writes its rows, and its own spill at the pause spills nothing on the helper's turn. */ +/* txn_spill_tests_rewriter is the writers' lock case's merger: its transaction's two spills of halves of its rows, the second merging with the first. */ static bool -txn_spill_tests_owner(void *captures, HostIoIo *io, HostError *error) +txn_spill_tests_rewriter(void *captures, HostIoIo *io, HostError *error) { - HostStorage storage = txn_memtable_testing_storage(9494); + static const uint64 halves[2][2] = {{0, 10}, {10, 20}}; + HostStorage storage = txn_memtable_testing_storage(9513); + uint64 merging = *(const uint64 *) captures; HostSharedHeld state; - uint64 xid; - bool ok = true; - bool spilled; - if (!txn_memtable_testing_stated(io, 9494, &state, error)) + if (!txn_memtable_testing_stated(io, 9513, &state, error)) return false; - xid = simc_begin().f0; - for (uint64 number = 0; ok && number < 50; number++) + for (size_t k = 0; k < lengthof(halves); k++) { - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - size_t need = txn_memtable_version_encoded(&change.writer, change.op, change.shown, change.has_lsn, - change.lsn); - TxnMemtableVersionHandle handle; - uint64 touched; - uint8 key[8]; + bool spilled; - std_u64_to_be_bytes(number, key); - ok = txn_written_touched(io, xid, &storage, TXN_WRITTEN_OF_ROWS, key, sizeof(key), need, &touched, error) && - txn_memtable_table_push(io, host_shared_Held_record(&state), key, sizeof(key), &change, &handle, error); - txn_memtable_Op_drop(&op); + txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), &storage, merging, halves[k][0], + halves[k][1]); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, merging, &spilled, error)) + { + host_shared_Held_drop(&state); + return false; + } + STD_ASSERT(2934, 17, spilled, "spilled(io, &storage(9513), Of::Rows, merging)?"); } - ok = ok && host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-spill-case-written", error) && - txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, error); - if (ok) - STD_ASSERT_MSG(2615, 13, !spilled, "the owner spills nothing on the helper's turn"); - ok = ok && txn_written_ended(io, error); host_shared_Held_drop(&state); - return ok; + return true; } -/* txn_spill_tests_helper is the turn case's helper: its relief spills the owner's transaction's versions. */ +/* txn_spill_tests_lister is the writers' lock case's other transaction: it spills its own rows, its listing waiting on the merge's lock. */ static bool -txn_spill_tests_helper(void *captures, HostIoIo *io, HostError *error) +txn_spill_tests_lister(void *captures, HostIoIo *io, HostError *error) { - bool relieved; + HostStorage storage = txn_memtable_testing_storage(9513); + uint64 other = *(const uint64 *) captures; + HostSharedHeld state; + bool spilled; + bool ok; - if (!txn_spill_relieved(io, &relieved, error)) + if (!txn_memtable_testing_stated(io, 9513, &state, error)) return false; - STD_ASSERT(2619, 13, relieved, "relieved(io)?"); - return true; + txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), &storage, other, 100, 110); + ok = txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, other, &spilled, error); + if (ok) + STD_ASSERT(2941, 13, spilled, "spilled(io, &storage(9513), Of::Rows, other)?"); + host_shared_Held_drop(&state); + return ok; } /* - * Bugs row 144's turn: while a helper spills a transaction's versions, held - * inside its spill, the transaction's owner asks to spill its own tree and - * spills nothing, the turn being the helper's; once the helper is done, the - * transaction's files have distinct numbers and every key reads once. + * Bugs row 143's writers' lock: a merge holds its storage's manifest's + * writers' lock after it read the manifest, and another transaction's listing + * of its own spill of the same storage waits its turn, so both the merged + * file and the other's file are listed after, nothing lost, and every key + * reads back once. */ void -test_txn_spill_tests_an_owner_spills_nothing_while_a_helper_holds_its_transactions_turn(void) -{ - static const StdFmtPiece lead[] = {STD_FMT_LIT("the run didn't end: "), STD_FMT_ARG(0)}; - SimcStep steps[2] = {{"owner", 5, PG_UINT64_MAX}, {"helper", 6, PG_UINT64_MAX}}; - SimcActor owner = {txn_spill_tests_owner}; - SimcActor helper = {txn_spill_tests_helper}; +test_txn_spill_tests_a_merges_rewrite_and_another_transactions_listing_lose_nothing(void) +{ + static const StdFmtPiece lead[] = {STD_FMT_LIT("the run didn't end: "), STD_FMT_ARG(0)}; + static const uint64 numbers[] = {0, 19, 100, 109}; + SimcStep steps[1] = {{"other", 5, PG_UINT64_MAX}}; SimcTestIo test_io; HostIoIo *io; HostSharedHeld held; @@ -9004,999 +13408,1391 @@ test_txn_spill_tests_an_owner_spills_nothing_while_a_helper_holds_its_transactio SimcReport report; HostDeltaHold hold; HostError error; - StdVec files; - size_t count; - size_t distinct = 0; + StdVec xids; + uint64 merging; + uint64 other; + uint64 listed[2]; + SimcActor merger = {txn_spill_tests_rewriter, &merging}; + SimcActor lister = {txn_spill_tests_lister, &other}; simc_size_delta(64 << 20); test_io = simc_io(); io = &test_io.io; - held = txn_memtable_testing_created(io, 9494); + held = txn_memtable_testing_created(io, 9513); host_shared_Held_drop(&held); + merging = simc_begin().f0; + other = simc_begin().f0; run = simc_Run_new(1); - simc_Run_actor(&run, "owner", 5, owner); - simc_Run_actor(&run, "helper", 6, helper); - simc_Run_until(&run, "owner", 5, "doltgres-spill-case-written"); - simc_Run_until(&run, "helper", 6, "doltgres-spill-pended"); + simc_Run_actor(&run, "merger", 6, merger); + simc_Run_actor(&run, "other", 5, lister); + simc_Run_until(&run, "merger", 6, "doltgres-spill-merge-pended"); + simc_Run_until(&run, "merger", 6, "doltgres-spill-manifest-read"); simc_Run_script(&run, steps, lengthof(steps)); report = simc_Run_finish(run); - txn_spill_tests_returned(&report, 2626, 67, lead, 2627); - if (!txn_memtable_testing_stated(io, 9494, &held, &error)) - STD_RESULT_EXPECT_FAILED(2628, 38, "called `Result::unwrap()` on an `Err` value", &error, + txn_spill_tests_returned(&report, 2948, 67, lead, 2949); + if (!txn_memtable_testing_stated(io, 9513, &held, &error)) + STD_RESULT_EXPECT_FAILED(2950, 38, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - files = txn_spill_tests_files_of(io, host_shared_Held_record(&held)); + xids = txn_spill_tests_rows_xids(io, host_shared_Held_record(&held)); + listed[0] = other; + listed[1] = merging; + txn_spill_tests_xids_are(2956, &xids, listed, lengthof(listed)); if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2631, 36, "called `Result::unwrap()` on an `Err` value", &error, + STD_RESULT_EXPECT_FAILED(2957, 36, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - count = files.len; - for (size_t k = 0; k < files.len; k++) - { - const TxnSpillFile *file = &STD_VEC_AT(&files, TxnSpillFile, k); - bool seen = false; - - for (size_t j = 0; !seen && j < k; j++) - seen = STD_VEC_AT(&files, TxnSpillFile, j).kind == file->kind && - STD_VEC_AT(&files, TxnSpillFile, j).number == file->number; - if (!seen) - distinct++; - } - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - STD_ASSERT_EQ_MSG(2635, 9, distinct == count, &distinct, std_usize_Debug_fmt, &count, std_usize_Debug_fmt, - "no two files share a number"); - txn_spill_tests_one_version(2637, 13, 90, io, &hold, host_shared_Held_record(&held), 0, false); - txn_spill_tests_one_version(2637, 13, 90, io, &hold, host_shared_Held_record(&held), 49, false); + for (size_t k = 0; k < lengthof(numbers); k++) + txn_spill_tests_one_version(2959, 13, 90, io, &hold, host_shared_Held_record(&held), numbers[k], true); host_delta_Hold_drop(&hold); host_shared_Held_drop(&held); simc_Report_drop(&report); simc_TestIo_drop(&test_io); } -/* - * Bugs row 143's one file: a spill of more keys than one page of a filter - * holds writes one row file and one tuple-id file, each filter in several - * pages of its file, and every key reads back through the spill, its row's - * tuple id finding it through the tuple-id filter. - */ -void -test_txn_spill_tests_a_spill_of_many_keys_writes_one_file_whose_filter_takes_pages(void) +/* txn_spill_tests_paused_reader is the whole lists case's reader: it reads the key's versions, paused between its open of the manifest and its read. */ +static bool +txn_spill_tests_paused_reader(void *captures, HostIoIo *io, HostError *error) { - static const uint64 numbers[] = {0, 3000, 6000, 6999}; - static const TxnRunsKind wanted[] = {TXN_RUNS_KIND_SPILLED_TUPLES, TXN_RUNS_KIND_SPILLED_ROWS}; - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; - HostSharedRecord state; - HostStorage storage = txn_memtable_testing_storage(9483); - StdVec files; + HostSharedHeld state; HostDeltaHold hold; - HostError error; - uint64 xid; - bool spilled; - bool all = true; + StdVec versions; + size_t one = 1; + uint8 key[8]; + bool ok; - simc_size_delta(64 << 20); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9483); - state = host_shared_Held_record(&held); - xid = simc_begin().f0; - txn_spill_tests_tupled_rows(2656, 74, 2662, 51, io, state, &storage, xid, 0, 7000); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2664, 60, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2664, 9, spilled, "spilled(io, &storage(9483), Of::Rows, xid).unwrap()"); - files = txn_spill_tests_files_of(io, state); - txn_spill_tests_kinds_are(2667, &files, wanted, lengthof(wanted)); - for (size_t k = 0; all && k < files.len; k++) - all = STD_VEC_AT(&files, TxnSpillFile, k).bits > TXN_SPILL_PAGE_BITS; - STD_ASSERT(2668, 9, all, "files.iter().all(|file| file.bits > PAGE_BITS)"); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2669, 36, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - for (size_t k = 0; k < lengthof(numbers); k++) + if (!txn_memtable_testing_stated(io, 9514, &state, error)) + return false; + if (!host_delta_hold(io, &hold, error)) { - txn_spill_tests_one_version(2672, 13, 64, io, &hold, state, numbers[k], true); - txn_spill_tests_tuple_key_is(2673, io, &hold, state, numbers[k]); + host_shared_Held_drop(&state); + return false; + } + std_u64_to_be_bytes(100, key); + ok = txn_memtable_table_versions(io, &hold, host_shared_Held_record(&state), key, sizeof(key), &versions, error); + if (ok) + { + STD_ASSERT_EQ_MSG(2984, 13, versions.len == 1, &versions.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt, + "the paused read finds the version once"); + txn_spill_tests_versions_free(&versions); } host_delta_Hold_drop(&hold); - if (!txn_written_ended(io, &error)) - STD_RESULT_EXPECT_FAILED(2676, 28, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - host_shared_Held_drop(&held); - simc_TestIo_drop(&test_io); + host_shared_Held_drop(&state); + return ok; +} + +/* txn_spill_tests_spiller is the whole lists case's spiller: its spill of the key's version renames a new list over the manifest. */ +static bool +txn_spill_tests_spiller(void *captures, HostIoIo *io, HostError *error) +{ + HostStorage storage = txn_memtable_testing_storage(9514); + bool spilled; + + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, *(const uint64 *) captures, &spilled, error)) + return false; + STD_ASSERT(2988, 13, spilled, "spilled(io, &storage(9514), Of::Rows, second)?"); + return true; } /* - * Bugs row 74's abort: an aborted transaction's spills leave its storages' - * chains with its take-back, so no reader reads its spilled versions, and - * another transaction's spill stays. + * Bugs row 143's whole lists: a reader that read a key's chain in the + * memtable pauses between its open of the storage's manifest file and its + * read, while a spill of the key's version renames a new list over the + * manifest; it finds the key's version once, through the chain it read, and + * a later read finds it once through the new list. */ void -test_txn_spill_tests_an_abort_takes_its_spills_out_of_the_chain(void) +test_txn_spill_tests_a_reader_paused_in_a_manifest_read_finds_each_version_once(void) { - SimcTestIo test_io = simc_io(); - HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9747); - HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9747); - uint64 mine = simc_begin().f0; - uint64 other = simc_begin().f0; - const struct - { - uint64 xid; - const char *key; - } pushes[] = {{other, "o"}, {mine, "m"}}; - HostXid aborted = {mine}; - StdVec files; - StdVec versions; + static const StdFmtPiece lead[] = {STD_FMT_LIT("the run didn't end: "), STD_FMT_ARG(0)}; + SimcStep steps[1] = {{"spiller", 7, PG_UINT64_MAX}}; + SimcActor reader = {txn_spill_tests_paused_reader}; + HostStorage storage = txn_memtable_testing_storage(9514); + SimcTestIo test_io; + HostIoIo *io; + HostSharedHeld held; + SimcRun run; + SimcReport report; HostDeltaHold hold; HostError error; - size_t one = 1; + StdVec files; + uint64 first; + uint64 second; + size_t rows = 0; + size_t two = 2; bool spilled; - bool all = true; - - for (size_t k = 0; k < lengthof(pushes); k++) - { - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - TxnMemtableVersionHandle handle; - uint64 touched; + SimcActor spiller = {txn_spill_tests_spiller, &second}; - if (!txn_written_touched(io, mine, &storage, TXN_WRITTEN_OF_ROWS, (const uint8 *) pushes[k].key, 1, 0, - &touched, &error)) - STD_RESULT_EXPECT_FAILED(2690, 74, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_memtable_table_push(io, state, (const uint8 *) pushes[k].key, 1, &change, &handle, &error)) - STD_RESULT_EXPECT_FAILED(2698, 50, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_memtable_Op_drop(&op); - } - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, other, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2700, 62, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2700, 9, spilled, "spilled(io, &storage(9747), Of::Rows, other).unwrap()"); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2701, 61, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2701, 9, spilled, "spilled(io, &storage(9747), Of::Rows, mine).unwrap()"); - simc_abort(aborted); - if (!txn_written_aborted(io, &error)) - STD_RESULT_EXPECT_FAILED(2703, 30, "called `Result::unwrap()` on an `Err` value", &error, + simc_size_delta(64 << 20); + test_io = simc_io(); + io = &test_io.io; + held = txn_memtable_testing_created(io, 9514); + first = simc_begin().f0; + second = simc_begin().f0; + txn_spill_tests_pushed_rows(io, host_shared_Held_record(&held), &storage, first, 0, 10); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, first, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(2976, 62, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - files = txn_spill_tests_files_of(io, state); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2705, 36, "called `Result::unwrap()` on an `Err` value", &error, + STD_ASSERT(2976, 9, spilled, "spilled(io, &storage(9514), Of::Rows, first).unwrap()"); + txn_spill_tests_pushed_rows(io, host_shared_Held_record(&held), &storage, second, 100, 101); + host_shared_Held_drop(&held); + run = simc_Run_new(1); + simc_Run_actor(&run, "reader", 6, reader); + simc_Run_actor(&run, "spiller", 7, spiller); + simc_Run_until(&run, "reader", 6, "doltgres-spill-manifest-opened"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_spill_tests_returned(&report, 2994, 67, lead, 2995); + if (!txn_memtable_testing_stated(io, 9514, &held, &error)) + STD_RESULT_EXPECT_FAILED(2996, 38, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - for (size_t k = 0; all && k < files.len; k++) - all = STD_VEC_AT(&files, TxnSpillFile, k).xid == other; - STD_ASSERT(2706, 9, all && files.len > 0, "left.iter().all(|&xid| xid == other) && !left.is_empty()"); + files = txn_spill_tests_files_of(io, host_shared_Held_record(&held)); + for (size_t k = 0; k < files.len; k++) + if (STD_VEC_AT(&files, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_ROWS) + rows++; std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - if (!txn_memtable_table_versions(io, &hold, state, (const uint8 *) "m", 1, &versions, &error)) - STD_RESULT_EXPECT_FAILED(2707, 57, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2707, 9, versions.len == 0, "table::versions(io, &hold, state, b\"m\").unwrap().is_empty()"); - txn_spill_tests_versions_free(&versions); - if (!txn_memtable_table_versions(io, &hold, state, (const uint8 *) "o", 1, &versions, &error)) - STD_RESULT_EXPECT_FAILED(2708, 60, "called `Result::unwrap()` on an `Err` value", &error, + STD_ASSERT_EQ(2998, 9, rows == two, &rows, std_usize_Debug_fmt, &two, std_usize_Debug_fmt); + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(2999, 36, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_spill_tests_one_version(3000, 9, 86, io, &hold, host_shared_Held_record(&held), 100, false); + if (!txn_written_ended(io, &error)) + STD_RESULT_EXPECT_FAILED(3001, 28, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT_EQ(2708, 9, versions.len == 1, &versions.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt); - txn_spill_tests_versions_free(&versions); host_delta_Hold_drop(&hold); host_shared_Held_drop(&held); + simc_Report_drop(&report); simc_TestIo_drop(&test_io); } -/* txn_spill_tests_u64s_Debug_fmt is a Vec's Debug. */ -static bool -txn_spill_tests_u64s_Debug_fmt(const void *value, StdFormatter *f) -{ - const StdVec *v = value; - StdDebugList list = std_fmt_debug_list(f); - - for (size_t k = 0; k < v->len; k++) - std_DebugList_entry(&list, &STD_VEC_AT(v, uint64, k), std_u64_Debug_fmt); - return std_DebugList_finish(&list); -} - -/* txn_spill_tests_rows_xids returns the xids of a state's storage's files of spilled rows, newest first, as the redo case's spilled closure does. */ -static StdVec -txn_spill_tests_rows_xids(HostIoIo *io, HostSharedRecord state) -{ - StdVec files = txn_spill_tests_files_of(io, state); - StdVec xids; - - std_vec_new(&xids, sizeof(uint64), _Alignof(uint64)); - for (size_t k = 0; k < files.len; k++) - { - const TxnSpillFile *file = &STD_VEC_AT(&files, TxnSpillFile, k); - - if (file->kind == TXN_RUNS_KIND_SPILLED_ROWS) - std_vec_push_Checked(&xids, &file->xid, sizeof(uint64), _Alignof(uint64)); - } - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - return xids; -} - -/* txn_spill_tests_xids_are asserts at its line that a Vec of xids is a list of them, and frees it. */ +/* txn_spill_tests_ones fills an installed node's bytes with ones, as the evicted filter case's install closure does. */ static void -txn_spill_tests_xids_are(uint32 line, StdVec *xids, const uint64 *wanted, size_t len) +txn_spill_tests_ones(void *captures, uint8 *out, size_t len) { - StdVec wanted_vec = {(void *) wanted, len, len}; - - STD_ASSERT_EQ(line, 9, xids->len == len && memcmp(xids->ptr, wanted, len * sizeof(uint64)) == 0, xids, - txn_spill_tests_u64s_Debug_fmt, &wanted_vec, txn_spill_tests_u64s_Debug_fmt); - std_vec_free_Checked(xids, sizeof(uint64), _Alignof(uint64)); + memset(out, 1, len); } /* - * Bugs row 74's redo: the startup process, which has no transaction of its - * own, spills the replayed transactions' trees when it finds the delta full, - * the largest first and whatever their size, as a backend spills its own - * (bugs row 124), until none is left. + * Bugs row 143's evicted filter: a lookup in a spill's file reads its + * filter's page into the node cache; once other nodes' installs take the + * cache's one frame, the page is gone from the cache, and the next lookup + * reads it from the file again and finds its key. */ void -test_txn_spill_tests_redo_over_budget_spills_the_largest_replayed_transaction(void) +test_txn_spill_tests_a_lookup_reads_an_evicted_filter_page_again(void) { + static const size_t counts[HOST_CACHE_CLASSES_LEN] = {1, 0, 0, 0, 0}; + const StdBytes keys[1] = {{(const uint8 *) "k", 1}}; SimcTestIo test_io = simc_io(); HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9749); - HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9749); - uint64 small = simc_begin().f0; - uint64 large = simc_begin().f0; - const struct - { - uint64 xid; - const char *key; - } pushes[] = {{small, "s"}, {large, "l"}}; - const uint64 first[] = {large}; - const uint64 both[] = {small, large}; - StdVec xids; + HostSharedHeld held = txn_memtable_testing_created(io, 9515); + StdPathBuf path; + TxnSpillFile file = txn_spill_tests_written(io, host_shared_Held_record(&held), 745, 0, keys, lengthof(keys), + &path); + HostCacheKey page; HostError error; - uint64 count; - bool relieved; + bool hit; + bool contains; - for (size_t k = 0; k < lengthof(pushes); k++) + simc_size_shared(counts); + if (!txn_spill_hits(io, &path, &file, (const uint8 *) "k", 1, &hit, &error)) + STD_RESULT_EXPECT_FAILED(3014, 48, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(3014, 9, hit, "hits(io, (&path, &file), b\"k\").unwrap()"); + page = txn_spill_filter_key(&path, &file, 0); + if (!host_cache_contains(io, HOST_CACHE_PLACEMENT_SERVER, &page, &contains, &error)) + STD_RESULT_EXPECT_FAILED(3016, 63, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(3016, 9, contains, "cache::contains(io, Placement::Server, &page).unwrap()"); + for (uint8 other = 0; other < 4; other++) { - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}, - .has_lsn = true, .lsn = 9}; - TxnMemtableVersionHandle handle; + uint8 address[HOST_CACHE_ADDRESS]; + uint8 locator[HOST_CACHE_LOCATOR] = {0}; + HostCacheKey key; + HostCacheRead read; - if (!txn_memtable_table_push(io, state, (const uint8 *) pushes[k].key, 1, &change, &handle, &error)) - STD_RESULT_EXPECT_FAILED(2730, 50, "called `Result::unwrap()` on an `Err` value", &error, + memset(address, other, sizeof(address)); + key = host_cache_key(1, address, locator); + if (!host_cache_install(io, HOST_CACHE_PLACEMENT_SERVER, &key, 8, txn_spill_tests_ones, NULL, NULL, &read, + &error)) + STD_RESULT_EXPECT_FAILED(3019, 86, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - txn_memtable_Op_drop(&op); + host_cache_Read_drop(&read); } - if (!txn_spill_replayed(io, &storage, TXN_WRITTEN_OF_ROWS, small, 10, &count, &error)) - STD_RESULT_EXPECT_FAILED(2732, 59, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_spill_replayed(io, &storage, TXN_WRITTEN_OF_ROWS, large, (size_t) TXN_SPILL_LOW, &count, &error)) - STD_RESULT_EXPECT_FAILED(2733, 69, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_spill_relieved(io, &relieved, &error)) - STD_RESULT_EXPECT_FAILED(2738, 30, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2738, 9, relieved, "relieved(io).unwrap()"); - xids = txn_spill_tests_rows_xids(io, state); - txn_spill_tests_xids_are(2739, &xids, first, lengthof(first)); - if (!txn_spill_relieved(io, &relieved, &error)) - STD_RESULT_EXPECT_FAILED(2740, 30, "called `Result::unwrap()` on an `Err` value", &error, + if (!host_cache_contains(io, HOST_CACHE_PLACEMENT_SERVER, &page, &contains, &error)) + STD_RESULT_EXPECT_FAILED(3021, 64, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT(2740, 9, relieved, "relieved(io).unwrap()"); - xids = txn_spill_tests_rows_xids(io, state); - txn_spill_tests_xids_are(2741, &xids, both, lengthof(both)); - if (!txn_spill_relieved(io, &relieved, &error)) - STD_RESULT_EXPECT_FAILED(2742, 31, "called `Result::unwrap()` on an `Err` value", &error, + STD_ASSERT_MSG(3021, 9, !contains, "the other node took the page's frame"); + if (!txn_spill_hits(io, &path, &file, (const uint8 *) "k", 1, &hit, &error)) + STD_RESULT_EXPECT_FAILED(3022, 48, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT(2742, 9, !relieved, "!relieved(io).unwrap()"); + STD_ASSERT(3022, 9, hit, "hits(io, (&path, &file), b\"k\").unwrap()"); + std_vec_free_Checked(&path.bytes, 1, 1); host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); } -/* txn_spill_tests_u32s_Debug_fmt is a Vec's Debug. */ -static bool -txn_spill_tests_u32s_Debug_fmt(const void *value, StdFormatter *f) -{ - const StdVec *v = value; - StdDebugList list = std_fmt_debug_list(f); - - for (size_t k = 0; k < v->len; k++) - std_DebugList_entry(&list, &STD_VEC_AT(v, uint32, k), std_u32_Debug_fmt); - return std_DebugList_finish(&list); -} - -/* txn_spill_tests_put_by pushes a transaction's put of a value to a key of a state's memtable, as the tiers case's push closure does. */ -static void -txn_spill_tests_put_by(HostIoIo *io, HostSharedRecord state, uint64 xid, const char *key, const char *value) -{ - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) value, strlen(value)); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, - .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; - TxnMemtableVersionHandle handle; - HostError error; - - if (!txn_memtable_table_push(io, state, (const uint8 *) key, strlen(key), &change, &handle, &error)) - STD_RESULT_EXPECT_FAILED(2764, 50, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_memtable_Op_drop(&op); -} - -/* txn_spill_tests_values_are asserts at its line that a key's versions' values in a state's memtable and spills are a list, newest first, as the tiers case's values closure reads them. */ -static void -txn_spill_tests_values_are(uint32 line, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, - const char *key, const char *const *wanted, size_t len) -{ - StdVec versions; - StdVec values; - StdVec expected; - HostError error; - - if (!txn_memtable_table_versions(io, hold, state, (const uint8 *) key, strlen(key), &versions, &error)) - STD_RESULT_EXPECT_FAILED(2778, 18, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - std_vec_with_capacity_Checked(&values, versions.len, sizeof(StdVec), _Alignof(StdVec)); - for (size_t k = 0; k < versions.len; k++) - { - TxnMemtableVersionRecord record = - txn_memtable_version_Version_record(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, k)); - StdVec value = txn_spill_tests_value(&record); - - std_vec_push_Checked(&values, &value, sizeof(StdVec), _Alignof(StdVec)); - } - txn_spill_tests_versions_free(&versions); - txn_memtable_table_tests_keys_of(wanted, len, &expected); - STD_ASSERT_EQ(line, 9, txn_memtable_table_tests_keys_eq(&values, &expected), &values, - txn_memtable_table_tests_keys_Debug_fmt, &expected, txn_memtable_table_tests_keys_Debug_fmt); - txn_memtable_table_tests_keys_free(&expected); - txn_memtable_table_tests_keys_free(&values); -} - /* - * Bugs row 74's tiers: a transaction's second spill of a storage's rows, no - * larger than its first, merges with it into one file, a key written again - * after the first spill holding its newer version first, and the merged - * files leave the chain. + * The free space map's spill: a spill of a storage's free space map takes + * out the blocks only its transaction wrote, and every reader finds a + * spilled block's entries below the memtable: its newest, a removal + * included, its takers, a seek past it, and a version by its WAL record's + * end. */ void -test_txn_spill_tests_a_second_spill_merges_with_the_first_keeping_its_versions_order(void) +test_txn_spill_tests_a_spilled_free_space_maps_entries_are_read_below_the_memtable(void) { - static const char *const a[] = {"1"}; - static const char *const b[] = {"3", "2"}; - static const char *const c[] = {"4"}; - static const uint32 merged[] = {2}; SimcTestIo test_io = simc_io(); HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9516); + HostSharedHeld held = txn_memtable_testing_created(io, 9762); HostSharedRecord state = host_shared_Held_record(&held); - HostStorage storage = txn_memtable_testing_storage(9516); + HostStorage storage = txn_memtable_testing_storage(9762); uint64 mine = simc_begin().f0; - StdVec files; - StdVec rows; - StdVec merged_vec = {(void *) merged, lengthof(merged), lengthof(merged)}; + uint64 other = simc_begin().f0; + const struct + { + uint64 xid; + uint32 block; + const char *value; + bool has_lsn; + uint64 lsn; + } pushes[] = {{mine, 7, "e", true, 4242}, {mine, 8, "f"}, {mine, 8, NULL}, {other, 9, "g"}, + {mine, 10, "h"}}; + const struct + { + uint32 block; + bool some; + const char *value; + } newest[] = {{7, true, "e"}, {8, true, NULL}, {9, true, "g"}, {10, true, "h"}, {11, false, NULL}}; + const struct + { + uint32 from; + bool some; + uint32 block; + const char *value; + } sought[] = {{0, true, 7, "e"}, {8, true, 8, NULL}, {9, true, 9, "g"}, {10, true, 10, "h"}, + {11, false, 0, NULL}}; HostDeltaHold hold; + StdVec versions; + StdVec takers; HostError error; bool spilled; + bool holds; - txn_spill_tests_put_by(io, state, mine, "a", "1"); - txn_spill_tests_put_by(io, state, mine, "b", "2"); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2768, 61, "called `Result::unwrap()` on an `Err` value", &error, + for (size_t k = 0; k < lengthof(pushes); k++) + { + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_DELETE}; + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}, + .has_lsn = pushes[k].has_lsn, .lsn = pushes[k].lsn}; + TxnMemtableVersionHandle handle; + uint8 key[4]; + + if (pushes[k].value != NULL) + op = txn_memtable_testing_put((const uint8 *) pushes[k].value, 1); + std_u32_to_be_bytes(pushes[k].block, key); + if (!txn_memtable_table_alone_State(io, state, true, TXN_MEMTABLE_MAP_MAP_FREE, key, sizeof(key), &change, + &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + } + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_FREE, mine, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT(2768, 9, spilled, "spilled(io, &storage(9516), Of::Rows, mine).unwrap()"); - txn_spill_tests_put_by(io, state, mine, "b", "3"); - txn_spill_tests_put_by(io, state, mine, "c", "4"); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, mine, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2771, 61, "called `Result::unwrap()` on an `Err` value", &error, + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage(9762), Of::Free, mine).unwrap()"); + if (!host_delta_hold(io, &hold, &error) || + !txn_memtable_map_versions(&TXN_MEMTABLE_MAP_FREE_TREE_OPS, io, &hold, state, 7, &versions, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT(2771, 9, spilled, "spilled(io, &storage(9516), Of::Rows, mine).unwrap()"); - files = txn_spill_tests_files_of(io, state); - std_vec_new(&rows, sizeof(uint32), _Alignof(uint32)); - for (size_t k = 0; k < files.len; k++) + STD_ASSERT(__LINE__, 9, versions.len == 0, "map::versions::(io, &hold, state, 7).unwrap().is_empty()"); + txn_spill_tests_versions_free(&versions); + host_delta_Hold_drop(&hold); + for (size_t k = 0; k < lengthof(newest); k++) + { + TxnMemtableMapEntry entry; + TxnMemtableMapEntry wanted = {.has_value = newest[k].value != NULL}; + bool some; + StdOptionRef left; + StdOptionRef right = {newest[k].some, STD_ITEM(&wanted, txn_memtable_map_tests_seen_Debug_fmt)}; + + if (wanted.has_value) + std_slice_to_vec_Checked(&wanted.value, (const uint8 *) newest[k].value, 1, 1, 1); + if (!txn_memtable_map_newest(io, state, TXN_MEMTABLE_MAP_MAP_FREE, newest[k].block, &some, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + left = (StdOptionRef) + { + some, STD_ITEM(&entry, txn_memtable_map_tests_seen_Debug_fmt) + }; + STD_ASSERT_EQ(__LINE__, 9, some == newest[k].some && (!some || txn_memtable_map_tests_seen_eq(&entry, &wanted)), + &left, std_option_Debug_fmt, &right, std_option_Debug_fmt); + if (some) + txn_memtable_map_tests_seen_drop(&entry); + txn_memtable_map_tests_seen_drop(&wanted); + } + if (!txn_memtable_map_takers(io, state, TXN_MEMTABLE_MAP_MAP_FREE, 8, &takers, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ_U64(__LINE__, 9, takers.len, 2); + for (size_t k = 0; k < takers.len; k++) { - const TxnSpillFile *file = &STD_VEC_AT(&files, TxnSpillFile, k); + const txn_TupleWriterRowEntryU64TupleVecU32Bool_t *taker = + &STD_VEC_AT(&takers, txn_TupleWriterRowEntryU64TupleVecU32Bool_t, k); + TxnMemtableMapEntry wanted = {.has_value = k == 0}; - if (file->kind == TXN_RUNS_KIND_SPILLED_ROWS) - std_vec_push_Checked(&rows, &file->number, sizeof(uint32), _Alignof(uint32)); + if (wanted.has_value) + std_slice_to_vec_Checked(&wanted.value, (const uint8 *) "f", 1, 1, 1); + STD_ASSERT_EQ_U64(__LINE__, 9, taker->f0.xid, mine); + STD_ASSERT_EQ(__LINE__, 9, txn_memtable_map_tests_seen_eq(&taker->f2, &wanted), &taker->f2, + txn_memtable_map_tests_seen_Debug_fmt, &wanted, txn_memtable_map_tests_seen_Debug_fmt); + txn_memtable_map_tests_seen_drop(&wanted); } - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2774, 36, "called `Result::unwrap()` on an `Err` value", &error, + txn_memtable_map_takers_drop(&takers); + for (size_t k = 0; k < lengthof(sought); k++) + { + txn_TupleU32Entry_t found; + TxnMemtableMapEntry wanted = {.has_value = sought[k].value != NULL}; + bool some; + + if (wanted.has_value) + std_slice_to_vec_Checked(&wanted.value, (const uint8 *) sought[k].value, 1, 1, 1); + if (!txn_memtable_map_seek(io, state, TXN_MEMTABLE_MAP_MAP_FREE, sought[k].from, &some, &found, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, some == sought[k].some, "map::seek(io, state, Map::Free, from).unwrap().is_some()"); + if (some) + { + STD_ASSERT_EQ_U64(__LINE__, 9, found.f0, sought[k].block); + STD_ASSERT_EQ(__LINE__, 9, txn_memtable_map_tests_seen_eq(&found.f1, &wanted), &found.f1, + txn_memtable_map_tests_seen_Debug_fmt, &wanted, txn_memtable_map_tests_seen_Debug_fmt); + txn_memtable_map_tests_seen_drop(&found.f1); + } + txn_memtable_map_tests_seen_drop(&wanted); + } + if (!txn_memtable_map_holds(io, state, TXN_MEMTABLE_MAP_MAP_FREE, 7, 4242, &holds, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT_EQ(2775, 9, rows.len == 1 && STD_VEC_AT(&rows, uint32, 0) == 2, &rows, txn_spill_tests_u32s_Debug_fmt, - &merged_vec, txn_spill_tests_u32s_Debug_fmt); - std_vec_free_Checked(&rows, sizeof(uint32), _Alignof(uint32)); - txn_spill_tests_values_are(2786, io, &hold, state, "a", a, lengthof(a)); - txn_spill_tests_values_are(2787, io, &hold, state, "b", b, lengthof(b)); - txn_spill_tests_values_are(2788, io, &hold, state, "c", c, lengthof(c)); - host_delta_Hold_drop(&hold); + STD_ASSERT(__LINE__, 9, holds, "map::holds(io, state, Map::Free, 7, 4242).unwrap()"); + if (!txn_memtable_map_holds(io, state, TXN_MEMTABLE_MAP_MAP_FREE, 7, 4243, &holds, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, !holds, "!map::holds(io, state, Map::Free, 7, 4243).unwrap()"); host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); } /* - * Bugs row 107's threshold (STORAGE_DESIGN §III.3): a spill's threshold is an - * eighth of the budget, and at the least delta_memory that is below LOW, so a - * recovery's replayed transactions there spill before the delta's fixed - * records leave no room for any of them to reach LOW. + * An aborted transaction's free space map spill, whose takes are never + * taken back: its abort and every checkpoint's naming keep it listed, it + * holds no repository's chunk spills back, and the fold's drop takes it out + * once a fold applied its takes; another transaction's helper at the budget + * spills a storage's free space map too. */ void -test_txn_spill_tests_a_small_budgets_threshold_is_an_eighth_of_it(void) +test_txn_spill_tests_an_aborted_free_space_map_spill_stays_listed_until_a_fold(void) { - uint64 threshold; - uint64 eighth = ((uint64) 1 << 20) / TXN_SPILL_SHARE; - - simc_size_delta(1 << 20); - threshold = txn_spill_threshold(); - STD_ASSERT_EQ(2798, 9, threshold == eighth, &threshold, std_u64_Debug_fmt, &eighth, std_u64_Debug_fmt); - STD_ASSERT(2799, 9, txn_spill_threshold() < TXN_SPILL_LOW, "threshold() < LOW"); -} + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9763); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9763); + HostXid aborting = simc_begin(); + HostXid other = simc_begin(); + const struct + { + uint64 xid; + uint32 block; + } pushes[] = {{aborting.f0, 7}, {other.f0, 9}}; + TxnMemtableMapEntry e = {.has_value = true}; + TxnMemtableMapEntry entry; + txn_TupleVecFileU64_t listed; + StdVec names; + HostError error; + bool spilled; + bool unfolded; + bool swept; + bool some; -/* txn_spill_tests_pushed_rows pushes a transaction's version of each of a range of numbered keys to a storage's rows, counted toward its spill as a writer's are. */ -static void -txn_spill_tests_pushed_rows(HostIoIo *io, HostSharedRecord state, const HostStorage *at, uint64 xid, uint64 start, - uint64 end) -{ - for (uint64 number = start; number < end; number++) + for (size_t k = 0; k < lengthof(pushes); k++) { - TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); - TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &op, + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "e", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].xid), .op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; TxnMemtableVersionHandle handle; - HostError error; - uint64 touched; - uint8 key[8]; + uint8 key[4]; - std_u64_to_be_bytes(number, key); - if (!txn_written_touched(io, xid, at, TXN_WRITTEN_OF_ROWS, key, sizeof(key), 0, &touched, &error)) - STD_RESULT_EXPECT_FAILED(2814, 62, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_memtable_table_push(io, state, key, sizeof(key), &change, &handle, &error)) - STD_RESULT_EXPECT_FAILED(2815, 51, "called `Result::unwrap()` on an `Err` value", &error, + op.u.put.heap_bytes = 0; + std_u32_to_be_bytes(pushes[k].block, key); + if (!txn_memtable_table_alone_State(io, state, true, TXN_MEMTABLE_MAP_MAP_FREE, key, sizeof(key), &change, + &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); txn_memtable_Op_drop(&op); } -} - -/* txn_spill_tests_relation returns the storage of a relation of a test's database, numbered past the first. */ -static HostStorage -txn_spill_tests_relation(uint32 database, uint32 number) -{ - HostStorage storage = txn_memtable_testing_storage(database); - - storage.relation = 16384 + number; - return storage; + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_FREE, aborting.f0, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage(9763), Of::Free, aborting.0).unwrap()"); + simc_abort(aborting); + if (!txn_spill_aborted(io, aborting.f0, &storage, 1, 150, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &listed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, listed.f0.len == 1 && + STD_VEC_AT(&listed.f0, TxnSpillFile, 0).kind == TXN_RUNS_KIND_SPILLED_FREE, + "the abort keeps the spill"); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (!txn_spill_named(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, 1, &names, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, names.len == 1 && STD_VEC_AT(&names, RepoRunFile, 0).kind == TXN_RUNS_KIND_SPILLED_FREE, + "names.iter().map(|file| file.kind).collect::>() == [Kind::SpilledFree as u8]"); + std_vec_free_Checked(&names, sizeof(RepoRunFile), _Alignof(RepoRunFile)); + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &listed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, listed.f0.len == 1 && + STD_VEC_AT(&listed.f0, TxnSpillFile, 0).kind == TXN_RUNS_KIND_SPILLED_FREE, + "a checkpoint's naming keeps the spill"); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + if (!txn_spill_unfolded_since(io, &held, 1, &unfolded, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, !unfolded, "a free space map spill holds no chunks back"); + if (!txn_spill_swept(io, other.f0, &swept, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, swept, "a helper spills another transaction's free space map"); + if (!txn_memtable_map_newest(io, state, TXN_MEMTABLE_MAP_MAP_FREE, 9, &some, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_slice_to_vec_Checked(&e.value, (const uint8 *) "e", 1, 1, 1); + STD_ASSERT(__LINE__, 9, some && txn_memtable_map_tests_seen_eq(&entry, &e), + "map::newest(io, state, Map::Free, 9).unwrap() == Some(Some(b\"e\".to_vec()))"); + txn_memtable_map_tests_seen_drop(&e); + if (some) + txn_memtable_map_tests_seen_drop(&entry); + if (!txn_spill_folded(io, state, 1, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &listed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, listed.f0.len == 1 && STD_VEC_AT(&listed.f0, TxnSpillFile, 0).xid == other.f0, + "the fold's drop takes the aborted spill and keeps a running one's"); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); } /* - * Bugs row 143's many storages: one transaction spills more storages than a - * 256 KB delta holds in 1 KB slots, its spills' bookkeeping taking none of - * the delta, so the delta's used bytes after each later spill are no more - * than after its first, and every storage's rows read back through its - * spill. + * The budget's committed arm: a key a running transaction and a committed + * one spilled, a third, committed, writing it again in the memtable, is + * relieved by spilling the third's versions, the chain's oldest in the + * memtable, which read newest above the spilled ones; a key whose oldest + * memtable version a running transaction wrote keeps the committed version + * above it, and the arm then has nothing left to spill. */ void -test_txn_spill_tests_a_transaction_spilling_more_storages_than_the_budget_holds_pages_completes(void) +test_txn_spill_tests_the_committed_arm_spills_a_keys_oldest_committed_versions_above_its_spills(void) { - static const StdFmtPiece past[] = {STD_FMT_LIT("after "), STD_FMT_ARG(0), STD_FMT_LIT(" the delta uses "), - STD_FMT_ARG(1), STD_FMT_LIT(" past "), STD_FMT_ARG(2)}; - static const uint32 read[] = {0, 299}; - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9771); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9771); + HostXid running = simc_begin(); + HostXid spilled_committed = simc_begin(); + HostXid above = simc_begin(); + HostXid holder = {0}; + HostXid after = {0}; + struct + { + HostXid xid; + const char *key; + const char *value; + bool spills; + } pushes[] = {{running, "k", "1", true}, {spilled_committed, "k", "2", true}, {above, "k", "3"}, + {{0}, "m", "4"}, {{0}, "m", "5"}}; + struct + { + HostXid xid; + HostLsn end; + } commits[3]; + uint64 before[3]; + HostDeltaHold hold; + StdVec chain; + StdVec versions; HostError error; - uint64 xid; - uint64 first = 0; - bool has_first = false; + bool spilled; + bool relieved; - simc_size_delta(256 << 10); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9510); - host_shared_Held_drop(&held); - xid = simc_begin().f0; - for (uint32 number = 0; number < 300; number++) + for (size_t k = 0; k < lengthof(pushes); k++) { - HostStorage at = txn_spill_tests_relation(9510, number); - HostSharedHeld state; - bool spilled; - uint64 used; + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) pushes[k].value, 1); + TxnMemtableVersionChange change = {.op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, + .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; - if (!txn_memtable_testing_held(io, &at, &state, &error)) - STD_RESULT_EXPECT_FAILED(2837, 59, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), &at, xid, 0, 4); - if (!txn_spill_spilled(io, &at, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2839, 73, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2839, 13, spilled, "spilled(io, &relation(9510, number), Of::Rows, xid).unwrap()"); - if (!has_first) + if (k == 3) { - first = host_delta_used(); - has_first = true; + holder = simc_begin(); + after = simc_begin(); + pushes[3].xid = holder; + pushes[4].xid = after; } - used = first; - if (!(host_delta_used() <= used)) + change.writer = txn_memtable_testing_writer(pushes[k].xid.f0); + if (!txn_memtable_table_push(io, state, (const uint8 *) pushes[k].key, 1, &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + if (!pushes[k].spills) + continue; + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, pushes[k].xid.f0, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage(9771), Of::Rows, xid.0).unwrap()"); + } + commits[0].xid = spilled_committed; + commits[0].end = 100; + commits[1].xid = above; + commits[1].end = 200; + commits[2].xid = after; + commits[2].end = 300; + for (size_t k = 0; k < lengthof(commits); k++) + { + TxnStatusOutcome committed = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = commits[k].end}; + TxnStatusRoomed roomed; + TxnStatusEntry entry; + bool took; + bool has; + + if (!txn_status_take(io, commits[k].xid, committed, &roomed, &took, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_commit(commits[k].xid); + if (!txn_status_lookup(io, commits[k].xid, &has, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, has, "status::lookup(io, xid).unwrap().is_some()"); + before[k] = entry.count; + } + if (!txn_spill_relieved(io, &relieved, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, relieved, "the committed arm spills the third writer's versions"); + if (!host_delta_hold(io, &hold, &error) || + !txn_memtable_table_chain(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, &hold, state, (const uint8 *) "k", 1, &chain, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, chain.len == 0, "k left the memtable"); + txn_spill_tests_versions_free(&chain); + if (!txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, io, &hold, state, TXN_RUNS_KIND_SPILLED_ROWS, + (const uint8 *) "k", 1, NULL, &versions, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + { + const uint64 wanted[] = {above.f0, spilled_committed.f0, running.f0}; + bool eq = versions.len == lengthof(wanted); + + for (size_t k = 0; eq && k < versions.len; k++) { - uint64 now = host_delta_used(); - StdFmtArg args[3] = {{&number, std_u32_Display_fmt}, {&now, std_u64_Display_fmt}, - {&used, std_u64_Display_fmt}}; - StdFmtArguments message = {past, lengthof(past), args}; + TxnMemtableVersionRecord record = + txn_memtable_version_Version_record(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, k)); - std_panic_fmt(RUST_FILE, 2841, 13, &message); + eq = txn_memtable_version_Record_writer(&record).xid == wanted[k]; } - host_shared_Held_drop(&state); + STD_ASSERT_MSG(__LINE__, 9, eq, "newest first"); } - for (size_t k = 0; k < lengthof(read); k++) + txn_spill_tests_versions_free(&versions); + if (!txn_memtable_table_chain(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, &hold, state, (const uint8 *) "m", 1, &chain, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ_U64(__LINE__, 9, chain.len, 2); + txn_spill_tests_versions_free(&chain); + host_delta_Hold_drop(&hold); + for (size_t k = 0; k < lengthof(commits); k++) { - HostStorage at = txn_spill_tests_relation(9510, read[k]); - HostSharedHeld state; - HostDeltaHold hold; + TxnStatusEntry entry; + bool has; - if (!txn_memtable_testing_held(io, &at, &state, &error)) - STD_RESULT_EXPECT_FAILED(2844, 59, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2845, 40, "called `Result::unwrap()` on an `Err` value", &error, + if (!txn_status_lookup(io, commits[k].xid, &has, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - txn_spill_tests_one_version(2846, 13, 88, io, &hold, host_shared_Held_record(&state), 3, false); - host_delta_Hold_drop(&hold); - host_shared_Held_drop(&state); + STD_ASSERT_MSG(__LINE__, 9, has && entry.count == before[k], "the spilled versions keep their counts"); } - if (!txn_written_ended(io, &error)) - STD_RESULT_EXPECT_FAILED(2848, 28, "called `Result::unwrap()` on an `Err` value", &error, + if (!txn_spill_relieved(io, &relieved, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, !relieved, "nothing committed is left at a chain's bottom"); + host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); } /* - * Bugs row 143's two at once: two transactions spill the same storages - * crosswise, one first in a storage and the other first in the next, more - * storages than a 256 KB delta holds in 1 KB slots, both completing, and - * once both abort, every manifest lists nothing and the delta's used bytes - * are no more than after the first storage's spills. + * The committed arm's spill after a restart: the versions it spilled, named + * by a checkpoint, read back after a restart in the order they read before + * it, newest first above the older transactions' spilled versions, each + * counted once at its writer's entry. */ void -test_txn_spill_tests_two_transactions_spilling_the_same_storages_crosswise_complete(void) +test_txn_spill_tests_the_committed_arms_spill_reads_back_in_order_after_a_restart(void) { - static const StdFmtPiece past[] = {STD_FMT_LIT("the delta uses "), STD_FMT_ARG(0), STD_FMT_LIT(" past "), - STD_FMT_ARG(1)}; - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; - HostStorage storages[300]; + const TxnInsertPage page = {.bytes = 8160, .most = 291}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9772); + HostStorage storage = txn_memtable_testing_storage(9772); + HostXid running = simc_begin(); + HostXid spilled_committed = simc_begin(); + HostXid above = simc_begin(); + const HostXid pushes[] = {running, spilled_committed, above}; + const char *const values[] = {"1", "2", "3"}; + const HostXid committed[] = {spilled_committed, above}; + const HostLsn ends[] = {100, 200}; + const uint64 wanted[] = {above.f0, spilled_committed.f0, running.f0}; HostError error; - uint64 one; - uint64 two; - uint64 first = 0; - bool has_first = false; + bool relieved; - simc_size_delta(256 << 10); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9511); - host_shared_Held_drop(&held); - one = simc_begin().f0; - two = simc_begin().f0; - for (uint32 number = 0; number < lengthof(storages); number++) - storages[number] = txn_spill_tests_relation(9511, number); - for (size_t number = 0; number < lengthof(storages); number++) + for (size_t k = 0; k < lengthof(pushes); k++) { - const HostStorage *at = &storages[number]; - uint64 order[2] = {number % 2 == 0 ? one : two, number % 2 == 0 ? two : one}; - HostSharedHeld state; + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) values[k], 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(pushes[k].f0), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + bool spilled; - if (!txn_memtable_testing_held(io, at, &state, &error)) - STD_RESULT_EXPECT_FAILED(2866, 38, "called `Result::unwrap()` on an `Err` value", &error, + if (!txn_memtable_table_push(io, host_shared_Held_record(&held), (const uint8 *) "k", 1, &change, &handle, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), at, one, 0, 2); - txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), at, two, 2, 4); - for (size_t k = 0; k < lengthof(order); k++) + txn_memtable_Op_drop(&op); + if (k == 2) + continue; + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, pushes[k].f0, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage(9772), Of::Rows, xid.0).unwrap()"); + } + for (size_t k = 0; k < lengthof(committed); k++) + { + TxnStatusOutcome outcome = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = ends[k]}; + TxnStatusRoomed roomed; + bool took; + + if (!txn_status_take(io, committed[k], outcome, &roomed, &took, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_commit(committed[k]); + } + if (!txn_spill_relieved(io, &relieved, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_MSG(__LINE__, 9, relieved, "the committed arm spills the third writer's version"); + for (int restarted = 0; restarted < 2; restarted++) + { + HostDeltaHold hold; + StdVec chain; + StdVec versions; + bool eq; + + if (restarted) { - bool spilled; + host_shared_Held_drop(&held); + simc_checkpointed(UINT64CONST(0x100000a27)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000a28), &error) || + !txn_state_unlinked(io, &storage, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; k < lengthof(committed); k++) + { + bool removed; - if (!txn_spill_spilled(io, at, TXN_WRITTEN_OF_ROWS, order[k], &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2870, 56, "called `Result::unwrap()` on an `Err` value", &error, + if (!txn_status_remove(io, committed[k], &removed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, removed, "status::remove(io, xid).unwrap()"); + } + if (!txn_runs_loaded(io, &page, &error) || !txn_memtable_testing_stated(io, 9772, &held, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT(2870, 17, spilled, "spilled(io, at, Of::Rows, xid).unwrap()"); } - if (!has_first) + if (!host_delta_hold(io, &hold, &error) || + !txn_memtable_table_chain(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, &hold, host_shared_Held_record(&held), + (const uint8 *) "k", 1, &chain, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, chain.len == 0, "table::chain::(io, &hold, state, b\"k\").unwrap().is_empty()"); + txn_spill_tests_versions_free(&chain); + if (!txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, io, &hold, host_shared_Held_record(&held), + TXN_RUNS_KIND_SPILLED_ROWS, (const uint8 *) "k", 1, NULL, &versions, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + eq = versions.len == lengthof(wanted); + for (size_t k = 0; eq && k < versions.len; k++) { - first = host_delta_used(); - has_first = true; + TxnMemtableVersionRecord record = + txn_memtable_version_Version_record(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, k)); + + eq = txn_memtable_version_Record_writer(&record).xid == wanted[k]; } - host_shared_Held_drop(&state); + if (restarted) + STD_ASSERT_MSG(__LINE__, 9, eq, "the same order after the start"); + else + STD_ASSERT_MSG(__LINE__, 9, eq, "newest first"); + txn_spill_tests_versions_free(&versions); + host_delta_Hold_drop(&hold); } - if (!txn_spill_aborted(io, one, storages, lengthof(storages), &error)) - STD_RESULT_EXPECT_FAILED(2875, 41, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - if (!txn_spill_aborted(io, two, storages, lengthof(storages), &error)) - STD_RESULT_EXPECT_FAILED(2875, 41, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - for (size_t k = 0; k < 2; k++) + for (size_t k = 0; k < lengthof(committed); k++) { - const HostStorage *at = &storages[k == 0 ? 0 : 299]; - HostSharedHeld state; - StdVec files; + TxnStatusEntry entry; + bool has; - if (!txn_memtable_testing_held(io, at, &state, &error)) - STD_RESULT_EXPECT_FAILED(2878, 38, "called `Result::unwrap()` on an `Err` value", &error, + if (!txn_status_lookup(io, committed[k], &has, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - files = txn_spill_tests_files_of(io, host_shared_Held_record(&state)); - STD_ASSERT(2879, 13, files.len == 0, "files_of(io, state.record()).is_empty()"); - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - host_shared_Held_drop(&state); - } - if (!(host_delta_used() <= first)) - { - uint64 now = host_delta_used(); - StdFmtArg args[2] = {{&now, std_u64_Display_fmt}, {&first, std_u64_Display_fmt}}; - StdFmtArguments message = {past, lengthof(past), args}; - - std_panic_fmt(RUST_FILE, 2882, 9, &message); + STD_ASSERT(__LINE__, 9, has && entry.count == 1, "counts == [Some(1), Some(1)]"); } - if (!txn_written_ended(io, &error)) - STD_RESULT_EXPECT_FAILED(2883, 28, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); + host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); } /* - * Bugs row 143's many spillers: twenty transactions, more than any fixed - * count of a storage's spillers, each spill one storage at a delta another - * process filled, every one completing with no page of the delta, and every - * transaction's rows read back. + * The crash mark, run file field 8: a running transaction's free space map + * spill that a checkpoint named, ended by a crash with no abort record, is + * marked by the next checkpoint's naming with that checkpoint's redo point, + * the point a fold passes it at, which a later start loads back with the + * file. */ void -test_txn_spill_tests_more_transactions_than_any_fixed_count_spill_one_storage_at_a_full_budget(void) +test_txn_spill_tests_a_free_space_spill_a_crash_ended_is_marked_with_the_next_checkpoints_redo_point(void) { - static const uint64 numbers[] = {0, 99, 199}; - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; - HostSharedRecord state; - HostStorage storage = txn_memtable_testing_storage(9512); - uint64 xids[20]; - StdVec filler; - StdVec files; - HostDeltaHold hold; + const TxnInsertPage page = {.bytes = 8160, .most = 291}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9773); + HostStorage storage = txn_memtable_testing_storage(9773); + HostXid xid = simc_begin(); + TxnFreeSpaceFree space = txn_free_space_Free_new(8000, NULL, 0, 291); + StdVec value; + TxnMemtableOp op; + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid.f0), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + uint8 key[4]; HostError error; - size_t rows = 0; - size_t count = lengthof(xids); + bool spilled; - simc_size_delta(2 << 20); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9512); - state = host_shared_Held_record(&held); - for (size_t k = 0; k < lengthof(xids); k++) - xids[k] = simc_begin().f0; - for (uint64 at = 0; at < lengthof(xids); at++) - txn_spill_tests_pushed_rows(io, state, &storage, xids[at], at * 10, at * 10 + 10); - std_vec_new(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned)); - if (!txn_spill_tests_fill(io, &filler, &error)) - STD_RESULT_EXPECT_FAILED(2902, 88, "called `Result::unwrap()` on an `Err` value", &error, + std_vec_new(&value, 1, 1); + txn_free_space_Free_encode_into(&space, &value); + op = txn_memtable_testing_put(value.ptr, value.len); + op.u.put.tuple.f0 = 7 << 16 | 1; + op.u.put.heap_bytes = 0; + std_vec_free_Checked(&value, 1, 1); + std_u32_to_be_bytes(7, key); + if (!txn_memtable_table_alone_State(io, host_shared_Held_record(&held), true, TXN_MEMTABLE_MAP_MAP_FREE, key, + sizeof(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - for (size_t k = 0; k < lengthof(xids); k++) + txn_memtable_Op_drop(&op); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_FREE, xid.f0, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage(9773), Of::Free, xid).unwrap()"); + host_shared_Held_drop(&held); + simc_checkpointed(UINT64CONST(0x100000a27)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000a28), &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (int round = 0; round < 3; round++) { - bool spilled; + static const HostLsn marks[] = {0, UINT64CONST(0x100000b28), UINT64CONST(0x100000b28)}; + static const char *const messages[] = {"named unmarked while its transaction ran", + "the crash's end marked at the next naming", "the mark loads back"}; + txn_TupleVecFileU64_t listed; + size_t found = 0; + bool marked = true; - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xids[k], &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2906, 64, "called `Result::unwrap()` on an `Err` value", &error, + if (round == 1) + { + simc_abort(xid); + simc_checkpointed(UINT64CONST(0x100000b27)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000b28), &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + } + else if (!txn_state_unlinked(io, &storage, &error) || !txn_runs_loaded(io, &page, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - STD_ASSERT(2906, 13, spilled, "spilled(io, &storage(9512), Of::Rows, xid).unwrap()"); + if (!txn_memtable_testing_stated(io, 9773, &held, &error) || + !txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, host_shared_Held_record(&held), &storage, &listed, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; k < listed.f0.len; k++) + { + const TxnSpillFile *file = &STD_VEC_AT(&listed.f0, TxnSpillFile, k); + + if (file->xid != xid.f0) + continue; + found++; + marked &= file->passed == marks[round]; + } + if (found != 1 || !marked) + std_panic(RUST_FILE, __LINE__, 9, messages[round], strlen(messages[round])); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + host_shared_Held_drop(&held); } - files = txn_spill_tests_files_of(io, state); - for (size_t k = 0; k < files.len; k++) - if (STD_VEC_AT(&files, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_ROWS) - rows++; - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - STD_ASSERT_EQ(2909, 9, rows == count, &rows, std_usize_Debug_fmt, &count, std_usize_Debug_fmt); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2910, 36, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - for (size_t k = 0; k < lengthof(numbers); k++) - txn_spill_tests_one_version(2912, 13, 81, io, &hold, state, numbers[k], true); - host_delta_Hold_drop(&hold); - std_vec_drop_Checked(&filler, sizeof(HostDeltaOwned), _Alignof(HostDeltaOwned), host_delta_Owned_drop); - if (!txn_written_ended(io, &error)) - STD_RESULT_EXPECT_FAILED(2915, 28, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); } -/* txn_spill_tests_rewriter is the writers' lock case's merger: its transaction's two spills of halves of its rows, the second merging with the first. */ -static bool -txn_spill_tests_rewriter(void *captures, HostIoIo *io, HostError *error) +/* + * A prepared transaction's spills: its rows' and its free space map's, its + * entry unowned once it prepared, are kept by a fold's drop at any point, as + * a commit it hasn't made can't apply, named by a checkpoint, and loaded + * back and read by a start, never taken as a committed or an aborted + * transaction's. + */ +void +test_txn_spill_tests_a_prepared_transactions_spills_are_never_taken_out(void) { - static const uint64 halves[2][2] = {{0, 10}, {10, 20}}; - HostStorage storage = txn_memtable_testing_storage(9513); - uint64 merging = *(const uint64 *) captures; - HostSharedHeld state; - - if (!txn_memtable_testing_stated(io, 9513, &state, error)) - return false; - for (size_t k = 0; k < lengthof(halves); k++) + const TxnInsertPage page = {.bytes = 8160, .most = 291}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9774); + HostStorage storage = txn_memtable_testing_storage(9774); + HostXid xid = simc_begin(); + TxnFreeSpaceFree space = txn_free_space_Free_new(8000, NULL, 0, 291); + TxnMemtableMapEntry entry = {.has_value = true}; + TxnMemtableOp row = txn_memtable_testing_put((const uint8 *) "r", 1); + TxnMemtableOp take; + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid.f0), .op = &row, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnStatusOutcome unowned = {.tag = TXN_STATUS_OUTCOME_UNOWNED}; + const TxnWrittenOf ofs[] = {TXN_WRITTEN_OF_ROWS, TXN_WRITTEN_OF_FREE}; + TxnStatusRoomed roomed; + TxnMemtableVersionHandle handle; + uint8 key[4]; + HostError error; + bool took; + + std_vec_new(&entry.value, 1, 1); + txn_free_space_Free_encode_into(&space, &entry.value); + take = txn_memtable_testing_put(entry.value.ptr, entry.value.len); + take.u.put.tuple.f0 = 7 << 16 | 1; + take.u.put.heap_bytes = 0; + if (!txn_memtable_table_push(io, host_shared_Held_record(&held), (const uint8 *) "k", 1, &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + change.op = &take; + std_u32_to_be_bytes(7, key); + if (!txn_memtable_table_alone_State(io, host_shared_Held_record(&held), true, TXN_MEMTABLE_MAP_MAP_FREE, key, + sizeof(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&take); + txn_memtable_Op_drop(&row); + for (size_t k = 0; k < lengthof(ofs); k++) { bool spilled; - txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), &storage, merging, halves[k][0], - halves[k][1]); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, merging, &spilled, error)) + if (!txn_spill_spilled(io, &storage, ofs[k], xid.f0, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage(9774), of, xid).unwrap()"); + } + if (!txn_status_take(io, xid, unowned, &roomed, &took, &error) || + !txn_spill_folded(io, host_shared_Held_record(&held), PG_UINT64_MAX >> 2, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (int restarted = 0; restarted < 2; restarted++) + { + txn_TupleVecFileU64_t listed; + TxnMemtableMapEntry newest; + HostDeltaHold hold; + StdVec versions; + bool some; + bool kept; + + if (restarted) { - host_shared_Held_drop(&state); - return false; + host_shared_Held_drop(&held); + simc_checkpointed(UINT64CONST(0x100000a27)); + if (!txn_runs_checkpoint(io, UINT64CONST(0x100000a28), &error) || + !txn_state_unlinked(io, &storage, &error) || !txn_runs_loaded(io, &page, &error) || + !txn_memtable_testing_stated(io, 9774, &held, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); } - STD_ASSERT(2934, 17, spilled, "spilled(io, &storage(9513), Of::Rows, merging)?"); + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, host_shared_Held_record(&held), &storage, &listed, + &error) || + !host_delta_hold(io, &hold, &error) || + !txn_spill_versions(&TXN_SPILL_STATE_HOME_OPS, io, &hold, host_shared_Held_record(&held), + TXN_RUNS_KIND_SPILLED_ROWS, (const uint8 *) "k", 1, NULL, &versions, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + host_delta_Hold_drop(&hold); + if (!txn_memtable_map_newest(io, host_shared_Held_record(&held), TXN_MEMTABLE_MAP_MAP_FREE, 7, &some, &newest, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + kept = listed.f0.len == 3 && versions.len == 1 && some && txn_memtable_map_tests_seen_eq(&newest, &entry); + if (restarted) + STD_ASSERT_MSG(__LINE__, 9, kept, "named and loaded back"); + else + STD_ASSERT_MSG(__LINE__, 9, kept, "kept by the drop"); + if (some) + txn_memtable_map_tests_seen_drop(&newest); + txn_spill_tests_versions_free(&versions); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); } - host_shared_Held_drop(&state); - return true; -} - -/* txn_spill_tests_lister is the writers' lock case's other transaction: it spills its own rows, its listing waiting on the merge's lock. */ -static bool -txn_spill_tests_lister(void *captures, HostIoIo *io, HostError *error) -{ - HostStorage storage = txn_memtable_testing_storage(9513); - uint64 other = *(const uint64 *) captures; - HostSharedHeld state; - bool spilled; - bool ok; - - if (!txn_memtable_testing_stated(io, 9513, &state, error)) - return false; - txn_spill_tests_pushed_rows(io, host_shared_Held_record(&state), &storage, other, 100, 110); - ok = txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, other, &spilled, error); - if (ok) - STD_ASSERT(2941, 13, spilled, "spilled(io, &storage(9513), Of::Rows, other)?"); - host_shared_Held_drop(&state); - return ok; + txn_memtable_map_tests_seen_drop(&entry); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); } /* - * Bugs row 143's writers' lock: a merge holds its storage's manifest's - * writers' lock after it read the manifest, and another transaction's listing - * of its own spill of the same storage waits its turn, so both the merged - * file and the other's file are listed after, nothing lost, and every key - * reads back once. + * A free space map spill's merge joins its own files alone: a transaction + * whose rows spilled once, with their file of tuple ids, and whose free + * space map spilled twice, merges its two free space map files into one and + * leaves the rows' tuple ids as they were, listing no file of tuple ids no + * row file has, so every row's tuple id still finds its key. */ void -test_txn_spill_tests_a_merges_rewrite_and_another_transactions_listing_lose_nothing(void) +test_txn_spill_tests_a_free_space_spills_merge_leaves_the_rows_tuple_ids_alone(void) { - static const StdFmtPiece lead[] = {STD_FMT_LIT("the run didn't end: "), STD_FMT_ARG(0)}; - static const uint64 numbers[] = {0, 19, 100, 109}; - SimcStep steps[1] = {{"other", 5, PG_UINT64_MAX}}; - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; - SimcRun run; - SimcReport report; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9776); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(9776); + uint64 xid = simc_begin().f0; + TxnFreeSpaceFree space = txn_free_space_Free_new(8000, NULL, 0, 291); + static const uint32 blocks[] = {7, 8}; + txn_TupleVecFileU64_t listed; + StdVec value; + StdVec rows; + StdVec tuples; HostDeltaHold hold; HostError error; - StdVec xids; - uint64 merging; - uint64 other; - uint64 listed[2]; - SimcActor merger = {txn_spill_tests_rewriter, &merging}; - SimcActor lister = {txn_spill_tests_lister, &other}; + bool spilled; + size_t frees = 0; - simc_size_delta(64 << 20); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9513); - host_shared_Held_drop(&held); - merging = simc_begin().f0; - other = simc_begin().f0; - run = simc_Run_new(1); - simc_Run_actor(&run, "merger", 6, merger); - simc_Run_actor(&run, "other", 5, lister); - simc_Run_until(&run, "merger", 6, "doltgres-spill-merge-pended"); - simc_Run_until(&run, "merger", 6, "doltgres-spill-manifest-read"); - simc_Run_script(&run, steps, lengthof(steps)); - report = simc_Run_finish(run); - txn_spill_tests_returned(&report, 2948, 67, lead, 2949); - if (!txn_memtable_testing_stated(io, 9513, &held, &error)) - STD_RESULT_EXPECT_FAILED(2950, 38, "called `Result::unwrap()` on an `Err` value", &error, + for (uint64 n = 1; n <= 3; n++) + { + TxnMemtableOp row = txn_memtable_testing_put((const uint8 *) "row", 3); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &row, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + uint8 key[8]; + + row.u.put.tuple.f0 = 1 << 16 | n; + std_u64_to_be_bytes(n, key); + if (!txn_memtable_table_push(io, state, key, sizeof(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&row); + } + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, xid, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - xids = txn_spill_tests_rows_xids(io, host_shared_Held_record(&held)); - listed[0] = other; - listed[1] = merging; - txn_spill_tests_xids_are(2956, &xids, listed, lengthof(listed)); + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage(9776), Of::Rows, xid).unwrap()"); + std_vec_new(&value, 1, 1); + txn_free_space_Free_encode_into(&space, &value); + for (size_t k = 0; k < lengthof(blocks); k++) + { + TxnMemtableOp take = txn_memtable_testing_put(value.ptr, value.len); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid), .op = &take, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + uint8 key[4]; + + take.u.put.tuple.f0 = 7 << 16 | 1; + take.u.put.heap_bytes = 0; + std_u32_to_be_bytes(blocks[k], key); + if (!txn_memtable_table_alone_State(io, state, true, TXN_MEMTABLE_MAP_MAP_FREE, key, sizeof(key), &change, + &handle, &error) || + !txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_FREE, xid, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage(9776), Of::Free, xid).unwrap()"); + txn_memtable_Op_drop(&take); + } + std_vec_free_Checked(&value, 1, 1); + if (!txn_spill_listed(&TXN_SPILL_STATE_HOME_OPS, io, state, &storage, &listed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_new(&rows, sizeof(uint32), _Alignof(uint32)); + std_vec_new(&tuples, sizeof(uint32), _Alignof(uint32)); + for (size_t k = 0; k < listed.f0.len; k++) + { + const TxnSpillFile *file = &STD_VEC_AT(&listed.f0, TxnSpillFile, k); + + if (file->kind == TXN_RUNS_KIND_SPILLED_FREE) + frees++; + else if (file->kind == TXN_RUNS_KIND_SPILLED_ROWS) + std_vec_push_Checked(&rows, &file->number, sizeof(uint32), _Alignof(uint32)); + else if (file->kind == TXN_RUNS_KIND_SPILLED_TUPLES) + std_vec_push_Checked(&tuples, &file->number, sizeof(uint32), _Alignof(uint32)); + } + STD_ASSERT_MSG(__LINE__, 9, frees == 1, "the free space map's files merged"); + STD_ASSERT_MSG(__LINE__, 9, rows.len == tuples.len && + (rows.len == 0 || memcmp(rows.ptr, tuples.ptr, rows.len * sizeof(uint32)) == 0), + "no tuple ids without their row file"); + std_vec_free_Checked(&rows, sizeof(uint32), _Alignof(uint32)); + std_vec_free_Checked(&tuples, sizeof(uint32), _Alignof(uint32)); + std_vec_free_Checked(&listed.f0, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2957, 36, "called `Result::unwrap()` on an `Err` value", &error, + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - for (size_t k = 0; k < lengthof(numbers); k++) - txn_spill_tests_one_version(2959, 13, 90, io, &hold, host_shared_Held_record(&held), numbers[k], true); + for (uint64 n = 1; n <= 3; n++) + { + HostRow tid = {1 << 16 | n}; + uint8 key[8]; + StdVec found; + bool some; + + std_u64_to_be_bytes(n, key); + if (!txn_spill_tuple_key(io, &hold, state, tid, &some, &found, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, some && found.len == sizeof(key) && memcmp(found.ptr, key, sizeof(key)) == 0, + "tuple_key(io, &hold, state, Row(1 << 16 | n)).unwrap() == Some(n.to_be_bytes().to_vec())"); + if (some) + std_vec_free_Checked(&found, 1, 1); + } host_delta_Hold_drop(&hold); host_shared_Held_drop(&held); - simc_Report_drop(&report); simc_TestIo_drop(&test_io); } -/* txn_spill_tests_paused_reader is the whole lists case's reader: it reads the key's versions, paused between its open of the manifest and its read. */ -static bool -txn_spill_tests_paused_reader(void *captures, HostIoIo *io, HostError *error) + +/* txn_spill_tests_residual_t is a failed residual's captures: the io, the state, the kept version, the files listed, and whether it listed them. */ +typedef struct txn_spill_tests_residual_t { - HostSharedHeld state; - HostDeltaHold hold; - StdVec versions; - size_t one = 1; - uint8 key[8]; - bool ok; + HostIoIo *io; + HostSharedRecord state; + const TxnSpillKept *kept; + StdVec files; + bool listed; + bool must; +} txn_spill_tests_residual_t; - if (!txn_memtable_testing_stated(io, 9514, &state, error)) - return false; - if (!host_delta_hold(io, &hold, error)) - { - host_shared_Held_drop(&state); - return false; - } - std_u64_to_be_bytes(100, key); - ok = txn_memtable_table_versions(io, &hold, host_shared_Held_record(&state), key, sizeof(key), &versions, error); - if (ok) - { - STD_ASSERT_EQ_MSG(2984, 13, versions.len == 1, &versions.len, std_usize_Debug_fmt, &one, std_usize_Debug_fmt, - "the paused read finds the version once"); - txn_spill_tests_versions_free(&versions); - } - host_delta_Hold_drop(&hold); - host_shared_Held_drop(&state); - return ok; +/* txn_spill_tests_residual_body lists the kept version's residual at a point, an error answering it unlisted. */ +static void +txn_spill_tests_residual_body(void *captures) +{ + txn_spill_tests_residual_t *c = captures; + HostError error; + + c->listed = txn_spill_residual(c->io, c->state, 7, c->kept, 1, &c->files, &error); + if (!c->listed && c->must) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!c->listed) + host_Error_drop(&error); } -/* txn_spill_tests_spiller is the whole lists case's spiller: its spill of the key's version renames a new list over the manifest. */ -static bool -txn_spill_tests_spiller(void *captures, HostIoIo *io, HostError *error) +/* txn_spill_tests_entry_count returns an xid's status entry's count. */ +static uint64 +txn_spill_tests_entry_count(HostIoIo *io, HostXid xid) { - HostStorage storage = txn_memtable_testing_storage(9514); - bool spilled; + TxnStatusEntry entry; + HostError error; + bool has; - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, *(const uint64 *) captures, &spilled, error)) - return false; - STD_ASSERT(2988, 13, spilled, "spilled(io, &storage(9514), Of::Rows, second)?"); - return true; + if (!txn_status_lookup(io, xid, &has, &entry, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, has, "the writer has an entry"); + return entry.count; } /* - * Bugs row 143's whole lists: a reader that read a key's chain in the - * memtable pauses between its open of the storage's manifest file and its - * read, while a spill of the key's version renames a new list over the - * manifest; it finds the key's version once, through the chain it read, and - * a later read finds it once through the new list. + * The residual's count order (bugs row 291's review R1): a residual adds its + * versions' count at their writer's entry before the manifest's hold, and + * its pad gives it back unless the listing finished, so a residual that + * fails or panics at any step lists no file and leaves the entry's count + * where it was, and one that finishes lists its file with the count added. */ void -test_txn_spill_tests_a_reader_paused_in_a_manifest_read_finds_each_version_once(void) +test_txn_spill_tests_a_residual_whose_listing_fails_lists_nothing_and_gives_its_count_back(void) { - static const StdFmtPiece lead[] = {STD_FMT_LIT("the run didn't end: "), STD_FMT_ARG(0)}; - SimcStep steps[1] = {{"spiller", 7, PG_UINT64_MAX}}; - SimcActor reader = {txn_spill_tests_paused_reader}; - HostStorage storage = txn_memtable_testing_storage(9514); - SimcTestIo test_io; - HostIoIo *io; - HostSharedHeld held; - SimcRun run; - SimcReport report; - HostDeltaHold hold; + bool failed = false; + + for (uint64 nth = 0; nth < 200; nth++) + { + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, 9760); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid writer = simc_begin(); + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionWriter written = txn_memtable_testing_writer(writer.f0); + TxnMemtableVersionShown shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + TxnSpillKept kept = {.kind = TXN_RUNS_KIND_ROWS}; + txn_spill_tests_residual_t residual = {.io = io, .state = state, .kept = &kept, .must = nth == 0}; + TxnStatusRoomed roomed; + HostError error; + StdCaught caught; + StdVec listed; + uint64 before; + + if (!txn_status_roomed(io, writer, &roomed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_slice_to_vec_Checked(&kept.key, "a", 1, 1, 1); + std_vec_new(&kept.bytes, 1, 1); + txn_memtable_version_encode(&written, &op, TXN_MEMTABLE_LOCK_MODE_UPDATE, shown, false, 0, &kept.bytes); + before = txn_spill_tests_entry_count(io, writer); + simc_fail_allocation(nth); + if (!std_catch_unwind(txn_spill_tests_residual_body, &residual, &caught)) + std_Caught_drop_Checked(&caught); + simc_fail_allocation(0); + listed = txn_spill_tests_files_of(io, state); + if (!residual.listed) + { + failed = true; + STD_ASSERT_EQ_U64(__LINE__, 9, listed.len, 0); + STD_ASSERT_EQ_U64(__LINE__, 9, txn_spill_tests_entry_count(io, writer), before); + } + else + { + STD_ASSERT_EQ_U64(__LINE__, 9, listed.len, 1); + STD_ASSERT_EQ_U64(__LINE__, 9, txn_spill_tests_entry_count(io, writer), before + 1); + std_vec_free_Checked(&residual.files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + } + std_vec_free_Checked(&listed, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + std_vec_free_Checked(&kept.key, 1, 1); + std_vec_free_Checked(&kept.bytes, 1, 1); + txn_memtable_Op_drop(&op); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); + if (nth > 0 && residual.listed) + break; + } + STD_ASSERT(__LINE__, 9, failed, "a failed allocation failed the residual"); +} + + +/* txn_spill_tests_crashed_spill spills a crashed transaction's rows or free space map versions, one version, a key or a block. */ +static void +txn_spill_tests_crashed_spill(HostIoIo *io, HostSharedRecord state, const HostStorage *storage, HostXid xid, + TxnWrittenOf of) +{ + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "e", 1); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(xid.f0), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; HostError error; - StdVec files; - uint64 first; - uint64 second; - size_t rows = 0; - size_t two = 2; + bool pushed; bool spilled; - SimcActor spiller = {txn_spill_tests_spiller, &second}; + uint8 key[4]; - simc_size_delta(64 << 20); - test_io = simc_io(); - io = &test_io.io; - held = txn_memtable_testing_created(io, 9514); - first = simc_begin().f0; - second = simc_begin().f0; - txn_spill_tests_pushed_rows(io, host_shared_Held_record(&held), &storage, first, 0, 10); - if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_ROWS, first, &spilled, &error)) - STD_RESULT_EXPECT_FAILED(2976, 62, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(2976, 9, spilled, "spilled(io, &storage(9514), Of::Rows, first).unwrap()"); - txn_spill_tests_pushed_rows(io, host_shared_Held_record(&held), &storage, second, 100, 101); - host_shared_Held_drop(&held); - run = simc_Run_new(1); - simc_Run_actor(&run, "reader", 6, reader); - simc_Run_actor(&run, "spiller", 7, spiller); - simc_Run_until(&run, "reader", 6, "doltgres-spill-manifest-opened"); - simc_Run_script(&run, steps, lengthof(steps)); - report = simc_Run_finish(run); - txn_spill_tests_returned(&report, 2994, 67, lead, 2995); - if (!txn_memtable_testing_stated(io, 9514, &held, &error)) - STD_RESULT_EXPECT_FAILED(2996, 38, "called `Result::unwrap()` on an `Err` value", &error, + if (of == TXN_WRITTEN_OF_FREE) + { + op.u.put.heap_bytes = 0; + std_u32_to_be_bytes(7, key); + pushed = txn_memtable_table_alone_State(io, state, true, TXN_MEMTABLE_MAP_MAP_FREE, key, sizeof(key), &change, + &handle, &error); + } + else + pushed = txn_memtable_table_push(io, state, (const uint8 *) "a", 1, &change, &handle, &error); + if (!pushed || !txn_spill_spilled(io, storage, of, xid.f0, &spilled, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - files = txn_spill_tests_files_of(io, host_shared_Held_record(&held)); - for (size_t k = 0; k < files.len; k++) - if (STD_VEC_AT(&files, TxnSpillFile, k).kind == TXN_RUNS_KIND_SPILLED_ROWS) - rows++; - std_vec_free_Checked(&files, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); - STD_ASSERT_EQ(2998, 9, rows == two, &rows, std_usize_Debug_fmt, &two, std_usize_Debug_fmt); - if (!host_delta_hold(io, &hold, &error)) - STD_RESULT_EXPECT_FAILED(2999, 36, "called `Result::unwrap()` on an `Err` value", &error, + STD_ASSERT(__LINE__, 9, spilled, "the version spilled"); + txn_memtable_Op_drop(&op); +} + +/* + * txn_spill_tests_crashed_count spills a transaction's rows and its free + * space map versions, in an order, ends it by a crash, with no abort record, + * drops what a fold passes, and returns its entry's count after the drop. + */ +static uint64 +txn_spill_tests_crashed_count(uint32 database, bool rows_first) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_memtable_testing_created(io, database); + HostSharedRecord state = host_shared_Held_record(&held); + HostStorage storage = txn_memtable_testing_storage(database); + HostXid crashed = simc_begin(); + TxnStatusRoomed roomed; + StdVec listed; + HostError error; + uint64 count; + + if (!txn_status_roomed(io, crashed, &roomed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - txn_spill_tests_one_version(3000, 9, 86, io, &hold, host_shared_Held_record(&held), 100, false); - if (!txn_written_ended(io, &error)) - STD_RESULT_EXPECT_FAILED(3001, 28, "called `Result::unwrap()` on an `Err` value", &error, + txn_spill_tests_crashed_spill(io, state, &storage, crashed, rows_first ? TXN_WRITTEN_OF_ROWS : TXN_WRITTEN_OF_FREE); + txn_spill_tests_crashed_spill(io, state, &storage, crashed, rows_first ? TXN_WRITTEN_OF_FREE : TXN_WRITTEN_OF_ROWS); + simc_abort(crashed); + if (!txn_spill_folded(io, state, 1, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - host_delta_Hold_drop(&hold); + listed = txn_spill_tests_files_of(io, state); + STD_ASSERT_EQ_U64(__LINE__, 9, listed.len, 0); + std_vec_free_Checked(&listed, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); + count = txn_spill_tests_entry_count(io, crashed); host_shared_Held_drop(&held); - simc_Report_drop(&report); simc_TestIo_drop(&test_io); + return count; } -/* txn_spill_tests_ones fills an installed node's bytes with ones, as the evicted filter case's install closure does. */ +/* + * The crashed transaction's counts (bugs row 291's review Q1): a drop decides + * every transaction whose files it reads before it counts a file, so a + * transaction a crash ended, whose rows spill stays listed beside its free + * space map spill until a checkpoint names them, has both files' counts + * taken back whichever is listed first. + */ +void +test_txn_spill_tests_a_drop_counts_a_crashed_transactions_files_in_either_order(void) +{ + uint64 rows_first = txn_spill_tests_crashed_count(9764, true); + uint64 free_first = txn_spill_tests_crashed_count(9765, false); + + STD_ASSERT_EQ_U64(__LINE__, 9, rows_first, free_first); +} + + +/* txn_spill_tests_carry_t is a carry case's captures: the io, the state, its writers, and whether the carry finished. */ +typedef struct txn_spill_tests_carry_t +{ + HostIoIo *io; + const HostSharedHeld *held; + uint64 writers[3]; + bool carried; + HostError error; +} txn_spill_tests_carry_t; + +/* txn_spill_tests_carry_logged is the carry case's logged: one row version of each writer, each its own key, as a chain of the rows. */ +static bool +txn_spill_tests_carry_logged(void *captures, HostSharedLongIo *io, StdVec *chains, HostError *error) +{ + const txn_spill_tests_carry_t *c = captures; + TxnMemtableVersionShown shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}; + + for (size_t k = 0; k < lengthof(c->writers); k++) + { + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "v", 1); + TxnMemtableVersionWriter writer = txn_memtable_testing_writer(c->writers[k]); + TxnSpillChain chain = {.of = TXN_WRITTEN_OF_ROWS}; + txn_TupleU64VecU8_t version = {.f0 = c->writers[k]}; + uint8 key = (uint8) ('a' + k); + + op.u.put.tuple.f0 = (uint64) (k + 1); + std_slice_to_vec_Checked(&chain.key, &key, 1, 1, 1); + std_vec_new(&version.f1, 1, 1); + txn_memtable_version_encode(&writer, &op, TXN_MEMTABLE_LOCK_MODE_UPDATE, shown, false, 0, &version.f1); + std_vec_new(&chain.versions, sizeof(txn_TupleU64VecU8_t), _Alignof(txn_TupleU64VecU8_t)); + std_vec_push_Checked(&chain.versions, &version, sizeof(version), _Alignof(txn_TupleU64VecU8_t)); + std_vec_push_Checked(chains, &chain, sizeof(chain), _Alignof(TxnSpillChain)); + txn_memtable_Op_drop(&op); + } + return true; +} + +/* txn_spill_tests_carry_body carries the writers' versions, an error answering it unfinished. */ static void -txn_spill_tests_ones(void *captures, uint8 *out, size_t len) +txn_spill_tests_carry_body(void *captures) { - memset(out, 1, len); + txn_spill_tests_carry_t *c = captures; + + c->carried = txn_spill_carried(c->io, c->held, c->writers, lengthof(c->writers), txn_spill_tests_carry_logged, c, + &c->error); } /* - * Bugs row 143's evicted filter: a lookup in a spill's file reads its - * filter's page into the node cache; once other nodes' installs take the - * cache's one frame, the page is gone from the cache, and the next lookup - * reads it from the file again and finds its key. + * txn_spill_tests_carry_refused runs a carry of three writers' versions with + * one of its steps refused after a count of them, a page's open or an + * allocation, and checks it: a carry that failed lists no file and leaves + * every writer's count where it was, and one that finished lists its files + * with each writer's count one more; it answers whether the carry failed. */ -void -test_txn_spill_tests_a_lookup_reads_an_evicted_filter_page_again(void) +static bool +txn_spill_tests_carry_refused(bool opens, uint64 after) { - static const size_t counts[HOST_CACHE_CLASSES_LEN] = {1, 0, 0, 0, 0}; - const StdBytes keys[1] = {{(const uint8 *) "k", 1}}; SimcTestIo test_io = simc_io(); HostIoIo *io = &test_io.io; - HostSharedHeld held = txn_memtable_testing_created(io, 9515); - StdPathBuf path; - TxnSpillFile file = txn_spill_tests_written(io, host_shared_Held_record(&held), 745, 0, keys, lengthof(keys), - &path); - HostCacheKey page; + HostSharedHeld held = txn_memtable_testing_created(io, 9766); + HostSharedRecord state = host_shared_Held_record(&held); + txn_spill_tests_carry_t carry = {.io = io, .held = &held}; + uint64 before[3]; + StdCaught caught; + StdVec listed; HostError error; - bool hit; - bool contains; + bool panicked; - simc_size_shared(counts); - if (!txn_spill_hits(io, &path, &file, (const uint8 *) "k", 1, &hit, &error)) - STD_RESULT_EXPECT_FAILED(3014, 48, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(3014, 9, hit, "hits(io, (&path, &file), b\"k\").unwrap()"); - page = txn_spill_filter_key(&path, &file, 0); - if (!host_cache_contains(io, HOST_CACHE_PLACEMENT_SERVER, &page, &contains, &error)) - STD_RESULT_EXPECT_FAILED(3016, 63, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(3016, 9, contains, "cache::contains(io, Placement::Server, &page).unwrap()"); - for (uint8 other = 0; other < 4; other++) + for (size_t k = 0; k < lengthof(carry.writers); k++) { - uint8 address[HOST_CACHE_ADDRESS]; - uint8 locator[HOST_CACHE_LOCATOR] = {0}; - HostCacheKey key; - HostCacheRead read; + HostXid writer = simc_begin(); + TxnStatusRoomed roomed; - memset(address, other, sizeof(address)); - key = host_cache_key(1, address, locator); - if (!host_cache_install(io, HOST_CACHE_PLACEMENT_SERVER, &key, 8, txn_spill_tests_ones, NULL, NULL, &read, - &error)) - STD_RESULT_EXPECT_FAILED(3019, 86, "called `Result::unwrap()` on an `Err` value", &error, + carry.writers[k] = writer.f0; + if (!txn_status_roomed(io, writer, &roomed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); - host_cache_Read_drop(&read); + before[k] = txn_spill_tests_entry_count(io, writer); } - if (!host_cache_contains(io, HOST_CACHE_PLACEMENT_SERVER, &page, &contains, &error)) - STD_RESULT_EXPECT_FAILED(3021, 64, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT_MSG(3021, 9, !contains, "the other node took the page's frame"); - if (!txn_spill_hits(io, &path, &file, (const uint8 *) "k", 1, &hit, &error)) - STD_RESULT_EXPECT_FAILED(3022, 48, "called `Result::unwrap()` on an `Err` value", &error, - host_Error_Debug_fmt); - STD_ASSERT(3022, 9, hit, "hits(io, (&path, &file), b\"k\").unwrap()"); - std_vec_free_Checked(&path.bytes, 1, 1); + if (opens) + simc_refuse_any_open(after); + else + simc_fail_allocation(after); + panicked = !std_catch_unwind(txn_spill_tests_carry_body, &carry, &caught); + if (panicked) + std_Caught_drop_Checked(&caught); + simc_refuse_nothing(); + simc_fail_allocation(0); + listed = txn_spill_tests_files_of(io, state); + if (!carry.carried && !panicked) + host_Error_drop(&carry.error); + STD_ASSERT(__LINE__, 9, carry.carried ? listed.len > 0 : listed.len == 0, + "a carry lists its files when it finishes and none when it fails"); + for (size_t k = 0; k < lengthof(carry.writers); k++) + { + HostXid writer = {carry.writers[k]}; + + STD_ASSERT_EQ_U64(__LINE__, 9, txn_spill_tests_entry_count(io, writer), before[k] + (carry.carried ? 1 : 0)); + } + std_vec_free_Checked(&listed, sizeof(TxnSpillFile), _Alignof(TxnSpillFile)); host_shared_Held_drop(&held); simc_TestIo_drop(&test_io); + return !carry.carried; +} + +/* + * The carry's count order (bugs row 291's review R2): a carry resolves its + * writers' slots before its hold and adds each listed file's count after it, + * and a failed add takes the carry's files back out with the counts already + * added in one checkpoint window, so a carry that fails at any page's open + * or allocation, the adds' included, lists no file and leaves every + * writer's count where it was, and one that finishes counts each writer's + * versions. + */ +void +test_txn_spill_tests_a_carry_that_fails_at_any_step_lists_nothing_and_leaves_every_count(void) +{ + bool failed = false; + + for (uint64 after = 0; after < 400; after++) + { + bool refused = txn_spill_tests_carry_refused(true, after); + + failed |= refused; + if (!refused && after > 0) + break; + } + for (uint64 nth = 1; nth < 400; nth++) + if (!txn_spill_tests_carry_refused(false, nth)) + break; + STD_ASSERT(__LINE__, 9, failed, "a refused open failed the carry"); } #endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/state.c b/src/doltgres/txn/c/state.c index 0a83cf9deb0bd..96615d0ef427e 100644 --- a/src/doltgres/txn/c/state.c +++ b/src/doltgres/txn/c/state.c @@ -45,6 +45,7 @@ #include "doltgres/txn/free_space.h" #include "doltgres/txn/memtable/table.h" #include "doltgres/txn/record.h" +#include "doltgres/txn/runs.h" #include "doltgres/txn/spill.h" #include "doltgres/txn/state.h" #include "doltgres/txn/status.h" @@ -85,6 +86,38 @@ #define TXN_STATE_REACHED_SINCE (25 + TXN_FREE_SPACE_CATEGORIES) #define TXN_STATE_RECOUNTING (28 + TXN_FREE_SPACE_CATEGORIES) +/* + * TXN_STATE_BASE_END is BaseEnd, the latest end of the records that named the + * bases installed in the state, which a save flushes the WAL through before + * its manifest names the newest (bugs row 350). + */ +#define TXN_STATE_BASE_END (32 + TXN_FREE_SPACE_CATEGORIES) + +/* + * TXN_STATE_LOGGING is Logging's flag, which every change of the base + * generation holds, so an install that logs its base's record checks the + * generation, inserts the record with no lock held, and publishes the base + * under that generation (bugs row 350). + */ +#define TXN_STATE_LOGGING 0 + +/* TXN_STATE_LOGGING_PAUSES names the pause inside a logging install's checkpoint delay. */ +static const char *const TXN_STATE_LOGGING_PAUSES[] = {"doltgres-state-logging"}; + +/* + * TXN_STATE_COMMITTING_INDEX is Committing, one past the id of the process + * whose turn it is at spilling the state's committed versions, so no two + * spill one key's versions at once, or zero. + */ +#define TXN_STATE_COMMITTING_INDEX (29 + TXN_FREE_SPACE_CATEGORIES) + +/* + * TXN_STATE_NUMBERED is Numbered, the highest number a spill file of an ended + * transaction of the state took, by the committed arm or a fold's residual, + * which take their numbers through it, so two of them never take one number. + */ +#define TXN_STATE_NUMBERED (30 + TXN_FREE_SPACE_CATEGORIES) + /* The makers' atomics, Tokens and Generations. */ #define TXN_STATE_TOKENS 0 #define TXN_STATE_GENERATIONS 1 @@ -352,6 +385,23 @@ txn_state_generation(HostSharedRecord record) return txn_state_load(record, TXN_STATE_BASE_GENERATION); } +/* + * txn_state_moved_past reports whether an install moved a state's base + * generation past the one a reader took its base at, loaded after every read + * before it (bugs rows 240 and 241). An install moves the generation by an + * atomic add, which has full barrier semantics, under the state's lock, and + * its drop starts only after the install returns; the load follows a read + * barrier, so a reader that read the delta and then finds the generation + * unmoved read no version a newer base's drop took out. + */ +bool +txn_state_moved_past(HostSharedRecord record, uint64 generation) +{ + HostSharedOp op = {.tag = HOST_SHARED_OP_LOAD_ACQUIRE}; + + return host_shared_Record_atomic(record, TXN_STATE_BASE_GENERATION, op) != generation; +} + /* txn_state_find_frame_0_t is find's state across its pad: the sweep and the handle being looked at, while live. */ typedef struct txn_state_find_frame_0_t { @@ -605,6 +655,15 @@ typedef struct txn_state_reclaim_frame_0_t bool *result; HostError *error; bool ok; + + /* + * unsaved is the repository and own tablespace of the first idle filled + * state the sweep found unsaved, which a reclaim that frees nothing + * saves. + */ + bool has_unsaved; + uint32 unsaved; + uint32 unsaved_own; HostSharedGuard making; bool making_live; StdVec swept; @@ -652,8 +711,8 @@ txn_state_reclaim_pad_0(void *frame_) /* * txn_state_reclaim_looked looks at one location of reclaim's sweep under - * the makers' lock: it answers false on an error, and sets freed when it - * freed the state. + * the makers' lock: it answers false on an error, sets freed when it freed + * the state, and keeps the first idle unsaved state's repository. */ static bool txn_state_reclaim_looked(txn_state_reclaim_frame_0_t *frame, uint64 location, bool *undecided, bool *freed) @@ -689,6 +748,15 @@ txn_state_reclaim_looked(txn_state_reclaim_frame_0_t *frame, uint64 location, bo return false; if (clean && maps) txn_state_exchange(record, TXN_STATE_FOLD_DUE_INDEX, 0, 1); + if (!clean && !frame->has_unsaved) + { + HostStorage storage; + TxnStatePersistence persistence; + + frame->has_unsaved = txn_state_described(record, &storage, &persistence); + frame->unsaved = storage.database; + frame->unsaved_own = txn_state_own(record); + } if (clean && !memtable && !maps) { frame->look_live = false; @@ -716,6 +784,8 @@ txn_state_reclaim_body_0(void *frame_) txn_state_reclaim_frame_0_t *frame = frame_; HostSharedRecord makers; bool undecided = false; + bool took = false; + bool cleaned = false; std_vec_new(&frame->kept, sizeof(uint64), _Alignof(uint64)); frame->kept_live = true; @@ -789,7 +859,13 @@ txn_state_reclaim_body_0(void *frame_) frame->held_live = false; host_shared_Held_drop(&frame->held); } - *frame->result = undecided; + if (!undecided && frame->has_unsaved && + !txn_runs_save(frame->io, frame->unsaved, frame->unsaved_own, &took, &cleaned, frame->error)) + { + txn_state_reclaim_pad_0(frame); + return; + } + *frame->result = undecided || (frame->has_unsaved && (took || cleaned)); frame->ok = true; txn_state_reclaim_pad_0(frame); } @@ -800,7 +876,10 @@ txn_state_reclaim_body_0(void *frame_) * holds, and answers whether the make should look again: when it freed one * or met one another reclaim was looking at. A state kept only by a memtable * that holds nothing and no pin names has it closed once the makers' lock is - * released, and the make looks again. + * released, and the make looks again. With nothing freed, it saves the + * repository of an idle state no manifest holds, after releasing the makers' + * lock, and the make looks again when the save turned a state clean or took + * an identity version out (item 6's caller (2), bugs row 340). */ static bool txn_state_State_Pool_reclaim(HostIoIo *io, bool *result, HostError *error) @@ -814,9 +893,9 @@ txn_state_State_Pool_reclaim(HostIoIo *io, bool *result, HostError *error) /* TXN_STATE_BLOBS is State's blobs: its newest base record and the one a replayed fold's record named, BASE_BYTES each. */ static const size_t TXN_STATE_BLOBS[] = {TXN_STATE_BASE_BYTES, TXN_STATE_BASE_BYTES}; -/* TXN_STATE_STATE_POOL_OPS is State's Shape and Pool: two locks, its atomics, and two base records' blobs. */ +/* TXN_STATE_STATE_POOL_OPS is State's Shape and Pool: two locks, a flag, its atomics, and two base records' blobs. */ const HostSharedPoolOps TXN_STATE_STATE_POOL_OPS = { - .shape = {.id = UINT64CONST(0x74786e000002), .locks = 2, .atomics = 29 + TXN_FREE_SPACE_CATEGORIES, + .shape = {.id = UINT64CONST(0x74786e000002), .locks = 2, .flags = 1, .atomics = 33 + TXN_FREE_SPACE_CATEGORIES, .blobs = TXN_STATE_BLOBS, .blobs_len = lengthof(TXN_STATE_BLOBS)}, .name = {"storage state", 13}, .count = txn_state_State_Pool_count, @@ -1172,13 +1251,18 @@ txn_state_state(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *read return frame.ok; } -/* txn_state_base_record_frame_0_t is base_record's state across its pad: the state's shared guard and its blob, while live. */ +/* + * txn_state_base_record_frame_0_t is base_record's state across its pad: the + * base generation read with the base, when asked for, and the state's shared + * guard and its blob, while live. + */ typedef struct txn_state_base_record_frame_0_t { HostIoIo *io; - const HostSharedHeld *held; + HostSharedRecord record; bool *some; TxnBaseBase *result; + uint64 *generation; HostError *error; bool ok; HostSharedGuard guard; @@ -1211,12 +1295,14 @@ txn_state_base_record_body_0(void *frame_) { txn_state_base_record_frame_0_t *frame = frame_; - if (!host_shared_Record_lock_shared(host_shared_Held_record(frame->held), HOST_IO_IO_TAKES_SHORT(frame->io), - TXN_STATE_STATE_LOCK_INDEX, &frame->guard, frame->error)) + if (!host_shared_Record_lock_shared(frame->record, HOST_IO_IO_TAKES_SHORT(frame->io), TXN_STATE_STATE_LOCK_INDEX, + &frame->guard, frame->error)) return; frame->guard_live = true; frame->blob = host_shared_Guard_read_blob(&frame->guard, 0); frame->blob_live = true; + if (frame->generation != NULL) + *frame->generation = txn_state_generation(frame->record); frame->guard_live = false; host_shared_Guard_drop(&frame->guard); frame->ok = txn_state_unkept(frame->blob.ptr, frame->blob.len, txn_state_internal, frame->some, frame->result, @@ -1231,12 +1317,73 @@ txn_state_base_record_body_0(void *frame_) bool txn_state_base_record(HostIoIo *io, const HostSharedHeld *held, bool *some, TxnBaseBase *result, HostError *error) { - txn_state_base_record_frame_0_t frame = {.io = io, .held = held, .some = some, .result = result, .error = error}; + txn_state_base_record_frame_0_t frame = {.io = io, .record = host_shared_Held_record(held), .some = some, + .result = result, .error = error}; std_landing(&frame, txn_state_base_record_body_0, txn_state_base_record_pad_0); return frame.ok; } +/* txn_state_read_point returns the read point of a state's newest base, or 0 for a state with none, read under the state's lock. */ +bool +txn_state_read_point(HostIoIo *io, HostSharedRecord record, uint64 *result, HostError *error) +{ + bool some; + TxnBaseBase newest; + txn_state_base_record_frame_0_t frame = {.io = io, .record = record, .some = &some, .result = &newest, + .error = error}; + + std_landing(&frame, txn_state_base_record_body_0, txn_state_base_record_pad_0); + if (!frame.ok) + return false; + *result = some ? newest.read : 0; + if (some) + txn_base_Base_drop(&newest); + return true; +} + +/* txn_state_committed_taken takes a process's turn at spilling a state's committed versions, answering whether it took it. */ +bool +txn_state_committed_taken(HostSharedRecord record, uint64 id) +{ + return txn_state_exchange(record, TXN_STATE_COMMITTING_INDEX, 0, id + 1) == 0; +} + +/* txn_state_committed_given gives a process's turn at spilling a state's committed versions back. */ +void +txn_state_committed_given(HostSharedRecord record, uint64 id) +{ + txn_state_exchange(record, TXN_STATE_COMMITTING_INDEX, id + 1, 0); +} + +/* + * txn_state_numbered takes the number of a spill file of an ended transaction + * of a state: past past, one past the transaction's files of the kind listed + * or pended, and past every number such a file of the state took before. + */ +uint32 +txn_state_numbered(HostSharedRecord record, uint32 past) +{ + uint64 taken = txn_state_load(record, TXN_STATE_NUMBERED); + + for (;;) + { + uint64 number = Max(taken + 1, (uint64) past); + uint64 seen = txn_state_exchange(record, TXN_STATE_NUMBERED, taken, number); + + if (seen == taken) + return (uint32) number; + taken = seen; + } +} + +/* txn_state_delta_bytes returns the encoded bytes of every version in a state's delta. */ +uint64 +txn_state_delta_bytes(HostSharedRecord record) +{ + return txn_state_load(record, TXN_STATE_DELTA_BYTES); +} + /* * txn_state_newest returns a filled state's newest base record, read under * the state's lock: base_record's read, since the blob read is a copy. @@ -1388,32 +1535,31 @@ txn_state_recounted_pad_0(void *frame_) * txn_state_recounted_counting runs recounted's count while the backend * holds the state's Recounting, answering false on the count's error, which * it gives back only after clearing RECOUNT and Recounting, and setting - * counted when the count answered sizes. + * counted when the count answered sizes. A count that a commit's move of the + * totals met leaves them unknown, and one a fold's install met counts again + * (bugs rows 240 and 316). */ static bool txn_state_recounted_counting(txn_state_recounted_frame_0_t *frame, HostSharedRecord record, HostSharedPlaceCell *cell, bool *counted) { uint64 moved; + uint64 generation; bool ok; TxnBaseSizes sizes; host_shared_PlaceCell_set(cell, true, host_shared_Held_place(frame->held)); moved = txn_state_load(record, TXN_STATE_TOTALS_MOVED); + generation = txn_state_generation(record); ok = host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-state-recounting", frame->error); - if (ok) + frame->count_live = false; + if (!ok && frame->count_drop != NULL) + frame->count_drop(frame->count_captures); + while (ok) { - frame->count_live = false; ok = frame->count(frame->count_captures, frame->io, counted, &sizes, frame->error); - } - else - { - frame->count_live = false; - if (frame->count_drop != NULL) - frame->count_drop(frame->count_captures); - } - if (ok && *counted) - { + if (!ok || !*counted) + break; txn_state_total_stored(record, 0, sizes.entries); txn_state_total_stored(record, 1, sizes.leaf_bytes); txn_state_total_stored(record, 2, sizes.key_bytes); @@ -1422,6 +1568,11 @@ txn_state_recounted_counting(txn_state_recounted_frame_0_t *frame, HostSharedRec txn_state_exchange(record, TXN_STATE_TOTALS_KNOWN, 0, 1); if (txn_state_load(record, TXN_STATE_TOTALS_MOVED) != moved) txn_state_totals_unknown(record); + if (!txn_state_moved_past(record, generation)) + break; + txn_state_totals_unknown(record); + moved = txn_state_load(record, TXN_STATE_TOTALS_MOVED); + generation = txn_state_generation(record); } host_shared_PlaceCell_set(cell, false, host_shared_Held_place(frame->held)); txn_state_exchange(record, TXN_STATE_RECOUNTING, 1, 0); @@ -2673,13 +2824,16 @@ txn_state_marked(HostIoIo *io, uint32 database, bool *result, HostError *error) bool txn_state_found(HostIoIo *io, const HostStorage *storage, bool *some, HostSharedHeld *result, HostError *error) { + if (!host_pause(HOST_SHARED_IO_PAUSES(io), "doltgres-state-found", error)) + return false; return txn_state_find(HOST_SHARED_IO_WITHIN(io), storage, some, result, error); } /* * txn_state_install_frame_0_t is the state of rechained and installed across - * their pad: the state's guard, its newest blob, the base it held, the base - * kept with its key, and the new blob, while live. + * their pad: the logging flag, the logged record's end and checkpoint delay, + * the state's guard, its newest blob, the base it held, the base kept with + * its key, and the new blob, while live. */ typedef struct txn_state_install_frame_0_t { @@ -2687,9 +2841,16 @@ typedef struct txn_state_install_frame_0_t const HostSharedHeld *held; uint64 read; const TxnBaseBase *base; + TxnStateLogged logged; + void *logged_captures; bool *result; HostError *error; bool ok; + HostSharedFlagGuard logging; + bool logging_live; + HostLsn end; + HostDelayCheckpoint window; + bool window_live; HostSharedGuard guard; bool guard_live; StdVec blob; @@ -2702,7 +2863,11 @@ typedef struct txn_state_install_frame_0_t bool bytes_live; } txn_state_install_frame_0_t; -/* txn_state_install_pad_0 is the installs' pad: the new blob, the bases, the blob, then the guard, while live. */ +/* + * txn_state_install_pad_0 is the installs' pad: the new blob, the bases, the + * blob, the guard, the checkpoint delay, whose drop unfinished stops the + * server, and the flag, while live. + */ static void txn_state_install_pad_0(void *frame_) { @@ -2733,30 +2898,77 @@ txn_state_install_pad_0(void *frame_) frame->guard_live = false; host_shared_Guard_drop(&frame->guard); } + if (frame->window_live) + { + frame->window_live = false; + host_DelayCheckpoint_drop(&frame->window); + } + if (frame->logging_live) + { + frame->logging_live = false; + host_shared_FlagGuard_drop(&frame->logging); + } +} + +/* + * txn_state_install_logged inserts a current logging install's record, with + * the state's lock released and taken again: outside recovery inside a + * checkpoint delay, which the install's write finishes, so an error before + * the base is published stops the server and recovery installs it from the + * record. The flag holds the generation meanwhile. + */ +static bool +txn_state_install_logged(txn_state_install_frame_0_t *frame) +{ + HostSharedRecord record = host_shared_Held_record(frame->held); + HostLsn reached = txn_state_load(record, TXN_STATE_REACHED_AT); + HostSharedFlagIo logging = host_shared_FlagGuard_io(&frame->logging); + HostIoIoDoors doors = HOST_SHARED_FLAG_IO_IO_DOORS(&logging); + + frame->guard_live = false; + host_shared_Guard_drop(&frame->guard); + if (!host_wal_stand(doors).recovering) + { + if (!host_delay_checkpoint(doors, TXN_STATE_LOGGING_PAUSES, lengthof(TXN_STATE_LOGGING_PAUSES), + &frame->window, frame->error)) + return false; + frame->window_live = true; + } + if (!frame->logged(frame->logged_captures, doors, reached, &frame->end, frame->error) || + !host_pause(HOST_SHARED_FLAG_IO_PAUSES(&logging), "doltgres-state-logging", frame->error) || + !host_shared_Record_lock(record, frame->io, TXN_STATE_STATE_LOCK_INDEX, &frame->guard, frame->error)) + return false; + frame->guard_live = true; + return true; } /* - * txn_state_install_locked takes an install's state's lock and compares its - * base generation with the one the caller read, answering false on an error - * and setting current when the generation is still the one read. + * txn_state_install_locked takes the state's logging flag, then an install's + * state's lock, and compares its base generation with the one the caller + * read, answering false on an error and setting current when the generation + * is still the one read; a current logging install then logs its record. */ static bool txn_state_install_locked(txn_state_install_frame_0_t *frame, bool *current) { HostSharedRecord record = host_shared_Held_record(frame->held); + if (!host_shared_Record_flag(record, frame->io, TXN_STATE_LOGGING, &frame->logging, frame->error)) + return false; + frame->logging_live = true; if (!host_shared_Record_lock(record, frame->io, TXN_STATE_STATE_LOCK_INDEX, &frame->guard, frame->error)) return false; frame->guard_live = true; *current = txn_state_load(record, TXN_STATE_BASE_GENERATION) == frame->read; *frame->result = false; - return true; + return !*current || frame->logged == NULL || txn_state_install_logged(frame); } /* * txn_state_install_written writes an install's new blob as the state's - * newest base, moving the generation and marking the state unsaved, - * answering false on an error. + * newest base, moving the generation, raising the state's base end to its + * record's, marking the state unsaved, and finishing a logging install's + * checkpoint delay, answering false on an error. */ static bool txn_state_install_written(txn_state_install_frame_0_t *frame) @@ -2766,14 +2978,64 @@ txn_state_install_written(txn_state_install_frame_0_t *frame) if (!host_shared_GuardExclusive_write_blob(&frame->guard, 0, frame->bytes.ptr, frame->bytes.len, frame->error)) return false; txn_state_add(record, TXN_STATE_BASE_GENERATION, 1); + if (frame->end > txn_state_load(record, TXN_STATE_BASE_END)) + txn_state_stored(record, TXN_STATE_BASE_END, frame->end); txn_state_unsaved(record); + if (frame->window_live) + { + frame->window_live = false; + host_DelayCheckpoint_finish(&frame->window); + } *frame->result = true; return true; } -/* txn_state_rechained_body_0 is rechained's body. */ +/* + * txn_state_install_kept encodes an install's new base as its blob, read + * against the state's newest under the lock: the new base takes the key of + * the newest it replaces when it doesn't know it, since a backend records a + * state's key in its newest base without moving the generation, after the + * install's caller read it, answering false on an error. + */ +static bool +txn_state_install_kept(txn_state_install_frame_0_t *frame) +{ + bool some; + bool keyed; + + frame->blob = host_shared_Guard_read_blob(&frame->guard, 0); + frame->blob_live = true; + if (!txn_state_unkept(frame->blob.ptr, frame->blob.len, txn_state_internal, &some, &frame->replaced, + frame->error)) + return false; + frame->replaced_live = some; + txn_state_kept(frame->base, &frame->bytes); + frame->bytes_live = true; + if (!some || frame->base->key.tag != TXN_BASE_KEY_UNKNOWN || frame->replaced.key.tag == TXN_BASE_KEY_UNKNOWN) + return true; + if (!txn_state_unkept(frame->bytes.ptr, frame->bytes.len, txn_state_internal, &keyed, &frame->keyed, + frame->error)) + return false; + if (!keyed) + { + *frame->error = txn_state_internal("an install's base record didn't keep", + strlen("an install's base record didn't keep")); + return false; + } + frame->keyed_live = true; + frame->bytes_live = false; + std_vec_free_Checked(&frame->bytes, 1, 1); + txn_base_Key_drop(&frame->keyed.key); + frame->keyed.key = frame->replaced.key; + frame->replaced.key.tag = TXN_BASE_KEY_UNKNOWN; + txn_state_kept(&frame->keyed, &frame->bytes); + frame->bytes_live = true; + return true; +} + +/* txn_state_install_body_0 is rechained's and installed's body. */ static void -txn_state_rechained_body_0(void *frame_) +txn_state_install_body_0(void *frame_) { txn_state_install_frame_0_t *frame = frame_; bool current; @@ -2783,17 +3045,7 @@ txn_state_rechained_body_0(void *frame_) txn_state_install_pad_0(frame); return; } - if (current) - { - txn_state_kept(frame->base, &frame->bytes); - frame->bytes_live = true; - if (!txn_state_install_written(frame)) - { - txn_state_install_pad_0(frame); - return; - } - } - frame->ok = true; + frame->ok = !current || (txn_state_install_kept(frame) && txn_state_install_written(frame)); txn_state_install_pad_0(frame); } @@ -2802,81 +3054,21 @@ txn_state_rechained_body_0(void *frame_) * differs only in its chain of pending transforms, keeping any predecessor, * under the state's lock when the base generation is still the one the * caller read, moving the generation and marking the state unsaved; it - * answers whether it replaced. + * answers whether it replaced. It takes a key recorded since the caller's + * read, as installed does. With logged, it inserts the new base's record + * first, as install_logged does. */ bool txn_state_rechained(HostIoTakesShort io, const HostSharedHeld *held, uint64 read, const TxnBaseBase *base, - bool *result, HostError *error) + TxnStateLogged logged, void *logged_captures, bool *result, HostError *error) { - txn_state_install_frame_0_t frame = {.io = io, .held = held, .read = read, .base = base, .result = result, - .error = error}; + txn_state_install_frame_0_t frame = {.io = io, .held = held, .read = read, .base = base, .logged = logged, + .logged_captures = logged_captures, .result = result, .error = error}; - std_landing(&frame, txn_state_rechained_body_0, txn_state_install_pad_0); + std_landing(&frame, txn_state_install_body_0, txn_state_install_pad_0); return frame.ok; } -/* txn_state_installed_body_0 is installed's body. */ -static void -txn_state_installed_body_0(void *frame_) -{ - txn_state_install_frame_0_t *frame = frame_; - bool current; - bool some; - - if (!txn_state_install_locked(frame, ¤t)) - { - txn_state_install_pad_0(frame); - return; - } - if (!current) - { - frame->ok = true; - txn_state_install_pad_0(frame); - return; - } - frame->blob = host_shared_Guard_read_blob(&frame->guard, 0); - frame->blob_live = true; - if (!txn_state_unkept(frame->blob.ptr, frame->blob.len, txn_state_internal, &some, &frame->replaced, - frame->error)) - { - txn_state_install_pad_0(frame); - return; - } - frame->replaced_live = some; - if (some && frame->base->key.tag == TXN_BASE_KEY_UNKNOWN && frame->replaced.key.tag != TXN_BASE_KEY_UNKNOWN) - { - bool keyed; - - txn_state_kept(frame->base, &frame->bytes); - frame->bytes_live = true; - if (!txn_state_unkept(frame->bytes.ptr, frame->bytes.len, txn_state_internal, &keyed, &frame->keyed, - frame->error)) - { - txn_state_install_pad_0(frame); - return; - } - if (!keyed) - { - *frame->error = txn_state_internal("a fold's base record didn't keep", - strlen("a fold's base record didn't keep")); - txn_state_install_pad_0(frame); - return; - } - frame->keyed_live = true; - frame->bytes_live = false; - std_vec_free_Checked(&frame->bytes, 1, 1); - txn_base_Key_drop(&frame->keyed.key); - frame->keyed.key = frame->replaced.key; - frame->replaced.key.tag = TXN_BASE_KEY_UNKNOWN; - txn_state_kept(&frame->keyed, &frame->bytes); - } - else - txn_state_kept(frame->base, &frame->bytes); - frame->bytes_live = true; - frame->ok = txn_state_install_written(frame); - txn_state_install_pad_0(frame); -} - /* * txn_state_installed makes a fold's base the state's newest, the newest it * replaces dropped, since every snapshot reads the newest, under the state's @@ -2885,16 +3077,17 @@ txn_state_installed_body_0(void *frame_) * save writes the new record; it answers whether it installed. The installed * base takes the key of the newest it replaces when the fold's base doesn't * know it, since a backend records a state's key in its newest base without - * moving the generation, after the fold read it. + * moving the generation, after the fold read it. With logged, it inserts the + * new base's record first, as install_logged does. */ bool txn_state_installed(HostIoTakesShort io, const HostSharedHeld *held, uint64 read, const TxnBaseBase *base, - bool *result, HostError *error) + TxnStateLogged logged, void *logged_captures, bool *result, HostError *error) { - txn_state_install_frame_0_t frame = {.io = io, .held = held, .read = read, .base = base, .result = result, - .error = error}; + txn_state_install_frame_0_t frame = {.io = io, .held = held, .read = read, .base = base, .logged = logged, + .logged_captures = logged_captures, .result = result, .error = error}; - std_landing(&frame, txn_state_installed_body_0, txn_state_install_pad_0); + std_landing(&frame, txn_state_install_body_0, txn_state_install_pad_0); return frame.ok; } @@ -3018,21 +3211,12 @@ txn_state_stored_base(HostIoIo *io, const RepoRepo *repo, const HostSharedReader } /* - * txn_state_readable returns the base of a filled state's that a snapshot - * reads: its newest, which every snapshot may read, since every fold's point - * is bounded by the storage's horizon, below every MVCC and historic - * snapshot's xmin on the primary, and a hot standby's replay of a built - * base cancels its snapshots at or below the base's maxFolded before it - * installs the base; a snapshot of any other kind reads the latest - * committed state. An MVCC or historic snapshot whose xmin doesn't pass the - * newest base's maxFolded is an internal error. + * txn_state_checked refuses, as an internal error, a base an MVCC or historic + * snapshot whose xmin doesn't pass its maxFolded would read. */ -bool -txn_state_readable(HostIoIo *io, const HostSharedHeld *held, const HostSnapshot *snapshot, bool *some, - TxnBaseBase *result, HostError *error) +static bool +txn_state_checked(const HostSnapshot *snapshot, bool *some, TxnBaseBase *result, HostError *error) { - if (!txn_state_newest(io, held, some, result, error)) - return false; if (*some && (snapshot->kind == HOST_SNAPSHOT_KIND_MVCC || snapshot->kind == HOST_SNAPSHOT_KIND_HISTORIC_MVCC) && snapshot->xmin.f0 <= result->max_folded) @@ -3046,6 +3230,46 @@ txn_state_readable(HostIoIo *io, const HostSharedHeld *held, const HostSnapshot return true; } +/* + * txn_state_readable returns the base of a filled state's that a snapshot + * reads: its newest, which every snapshot may read, since every fold's point + * is bounded by the storage's horizon, below every MVCC and historic + * snapshot's xmin on the primary, and a hot standby's replay of a built + * base cancels its snapshots at or below the base's maxFolded before it + * installs the base; a snapshot of any other kind reads the latest + * committed state. An MVCC or historic snapshot whose xmin doesn't pass the + * newest base's maxFolded is an internal error. + */ +bool +txn_state_readable(HostIoIo *io, const HostSharedHeld *held, const HostSnapshot *snapshot, bool *some, + TxnBaseBase *result, HostError *error) +{ + return txn_state_newest(io, held, some, result, error) && txn_state_checked(snapshot, some, result, error); +} + +/* + * txn_state_readable_since answers whether an install moved a filled state's + * base generation past the one a reader took its base at (bugs rows 240 and + * 241), costing one load when it hasn't, and when it has, returns the newest + * base for a snapshot to read, as readable returns it, with the generation + * it read it at, both read under the state's lock, under which an install + * moves both. + */ +bool +txn_state_readable_since(HostIoIo *io, HostSharedRecord record, const HostSnapshot *snapshot, uint64 generation, + bool *moved, bool *some, TxnBaseBase *result, uint64 *now, HostError *error) +{ + txn_state_base_record_frame_0_t frame = {.io = io, .record = record, .some = some, .result = result, + .generation = now, .error = error}; + + *moved = txn_state_generation(record) != generation; + *some = false; + if (!*moved) + return true; + std_landing(&frame, txn_state_base_record_body_0, txn_state_base_record_pad_0); + return frame.ok && txn_state_checked(snapshot, some, result, error); +} + /* txn_state_reset_frame_0_t is reset's state across its pad: the saved base it takes, the handle, the guard, the empty base, and its blob, while live. */ typedef struct txn_state_reset_frame_0_t { @@ -3218,15 +3442,14 @@ txn_state_uncounted_body_0(void *frame_) frame->versions_live = true; for (size_t k = 0, k_len = frame->versions.len; k < k_len; k++) { - TxnMemtableVersionRecord record = - txn_memtable_version_Version_record(&STD_VEC_AT(&frame->versions, TxnMemtableVersionVersion, k)); + const TxnMemtableVersionVersion *version = &STD_VEC_AT(&frame->versions, TxnMemtableVersionVersion, k); + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(version); TxnMemtableVersionWriter writer = txn_memtable_version_Record_writer(&record); - TxnMemtableVersionOpRefTag op = txn_memtable_version_Record_op(&record).tag; HostXid xid = {writer.has_top ? writer.top : writer.xid}; bool has; uint64 added; - if (op == TXN_MEMTABLE_VERSION_OP_REF_LOCK || op == TXN_MEMTABLE_VERSION_OP_REF_LOCK_ROW) + if (!txn_memtable_version_Version_counted(version)) continue; if (!txn_status_add(frame->io, xid, PG_UINT64_MAX, &has, &added, frame->error)) { @@ -3327,13 +3550,18 @@ typedef struct txn_state_emptied_frame_0_t const HostSharedHeld *held; HostError *error; bool ok; + HostSharedFlagGuard logging; + bool logging_live; HostDelayCheckpoint window; bool window_live; StdVec body; bool body_live; } txn_state_emptied_frame_0_t; -/* txn_state_emptied_pad_0 is emptied's pad: the body, then the window, while live. */ +/* + * txn_state_emptied_pad_0 is emptied's and redo_emptied's pad: the body, the + * window, then the logging flag, while live. + */ static void txn_state_emptied_pad_0(void *frame_) { @@ -3349,6 +3577,25 @@ txn_state_emptied_pad_0(void *frame_) frame->window_live = false; host_DelayCheckpoint_drop(&frame->window); } + if (frame->logging_live) + { + frame->logging_live = false; + host_shared_FlagGuard_drop(&frame->logging); + } +} + +/* + * txn_state_emptied_flagged takes the state's logging flag, which clearing a + * storage holds as an install does, since it moves the base generation. + */ +static bool +txn_state_emptied_flagged(txn_state_emptied_frame_0_t *frame) +{ + if (!host_shared_Record_flag(host_shared_Held_record(frame->held), HOST_IO_IO_TAKES_SHORT(frame->io), + TXN_STATE_LOGGING, &frame->logging, frame->error)) + return false; + frame->logging_live = true; + return true; } /* txn_state_emptied_body_0 is emptied's body. */ @@ -3359,10 +3606,13 @@ txn_state_emptied_body_0(void *frame_) TxnRecordStorageEmptied emptied; TxnStatePersistence persistence; - if (!txn_state_uncounted(frame->io, frame->held, frame->error) || + if (!txn_state_emptied_flagged(frame) || !txn_state_uncounted(frame->io, frame->held, frame->error) || !host_delay_checkpoint(HOST_IO_IO_IO_DOORS(frame->io), TXN_STATE_CLEARING, lengthof(TXN_STATE_CLEARING), &frame->window, frame->error)) + { + txn_state_emptied_pad_0(frame); return; + } frame->window_live = true; if (txn_state_described(host_shared_Held_record(frame->held), &emptied.storage, &persistence) && persistence == TXN_STATE_PERSISTENCE_PERMANENT) @@ -3388,14 +3638,15 @@ txn_state_emptied_body_0(void *frame_) frame->window_live = false; host_DelayCheckpoint_finish(&frame->window); frame->ok = true; + txn_state_emptied_pad_0(frame); } /* * txn_state_emptied empties a storage in place for a truncate of a * relfilenode its subtransaction made, which no other process reads or - * writes: it takes back its versions' counts, then, inside one checkpoint - * window, logs a permanent storage's emptying and clears the storage as - * cleared does. + * writes: under the logging flag, it takes back its versions' counts, then, + * inside one checkpoint window, logs a permanent storage's emptying and + * clears the storage as cleared does. */ bool txn_state_emptied(HostIoIo *io, const HostSharedHeld *held, HostError *error) @@ -3406,11 +3657,29 @@ txn_state_emptied(HostIoIo *io, const HostSharedHeld *held, HostError *error) return frame.ok; } -/* txn_state_redo_emptied replays a storage's emptying: it takes back its replayed versions' counts and clears it as cleared does. */ +/* txn_state_redo_emptied_body_0 is redo_emptied's body. */ +static void +txn_state_redo_emptied_body_0(void *frame_) +{ + txn_state_emptied_frame_0_t *frame = frame_; + + frame->ok = txn_state_emptied_flagged(frame) && txn_state_uncounted(frame->io, frame->held, frame->error) && + txn_state_cleared(frame->io, frame->held, frame->error); + txn_state_emptied_pad_0(frame); +} + +/* + * txn_state_redo_emptied replays a storage's emptying: under the logging + * flag, it takes back its replayed versions' counts and clears it as + * cleared does. + */ bool txn_state_redo_emptied(HostIoIo *io, const HostSharedHeld *held, HostError *error) { - return txn_state_uncounted(io, held, error) && txn_state_cleared(io, held, error); + txn_state_emptied_frame_0_t frame = {.io = io, .held = held, .error = error}; + + std_landing(&frame, txn_state_redo_emptied_body_0, txn_state_emptied_pad_0); + return frame.ok; } /* txn_state_unlinked_frame_0_t is unlinked's state across its pad: the makers' guard, the sweep, and the claimed state, while live. */ @@ -4099,13 +4368,31 @@ txn_state_unlogged(HostIoIo *io, StdVec *result, HostError *error) return frame.sweep.ok; } -/* txn_state_gathered_frame_0_t is gathered's frame: the slot reads' frame, the save's token, and the edit it gives. */ +/* + * txn_state_delta_clear reports whether a state's delta holds no version or + * take, in its memtables or in a spill file: its rows', free space map's, and + * tuple id table's words are clear, its spills' manifest lists no file, and + * no spill is pending. + */ +static bool +txn_state_delta_clear(HostSharedRecord record) +{ + return (txn_state_load(record, TXN_STATE_MEMTABLE_WORD_INDEX) | txn_state_load(record, TXN_STATE_FREE_WORD_INDEX) | + txn_state_load(record, TXN_STATE_TUPLES_WORD_INDEX) | txn_state_load(record, TXN_STATE_SPILLS_INDEX) | + txn_state_load(record, TXN_STATE_SPILLS_PENDING)) == 0; +} + +/* + * txn_state_gathered_frame_0_t is gathered's frame: the slot reads' frame, + * the save's token, and the edit and base end it gives. + */ typedef struct txn_state_gathered_frame_0_t { txn_state_slot_frame_0_t slot; uint64 token; bool *some; RepoGathered *result; + HostLsn *end; } txn_state_gathered_frame_0_t; /* txn_state_gathered_pad_0 is gathered's pad: the slot reads'. */ @@ -4143,8 +4430,7 @@ txn_state_gathered_body_0(void *frame_) txn_state_gathered_pad_0(frame); return; } - if ((txn_state_load(record, TXN_STATE_MEMTABLE_WORD_INDEX) | txn_state_load(record, TXN_STATE_FREE_WORD_INDEX) | - txn_state_load(record, TXN_STATE_TUPLES_WORD_INDEX)) == 0) + if (txn_state_delta_clear(record)) { txn_state_stored(record, TXN_STATE_COUNTER, some ? frame->slot.base.fresh_end : 1); if (!some) @@ -4178,6 +4464,7 @@ txn_state_gathered_body_0(void *frame_) frame->result->u.base.folded = frame->slot.base.folded; txn_base_Base_encode(&frame->slot.base, &frame->result->u.base.record.bytes, &frame->result->u.base.record.addresses); + *frame->end = txn_state_load(record, TXN_STATE_BASE_END); *frame->some = true; txn_state_gathered_pad_0(frame); } @@ -4186,21 +4473,22 @@ txn_state_gathered_body_0(void *frame_) * txn_state_gathered is one state's part of a save: under the state's lock, * for a ready state whose saved isn't clean, it first lets the tuple id * counter fall back to the next tuple id past every row its base record - * holds, (0,1) for no base, when no memtable of its rows or its free space - * map and no tuple id table is linked, since no version or entry then names - * a tuple id or a place past it and no insert has taken one; then it stores + * holds, (0,1) for no base, when its delta holds no version or take, in its + * memtables or in a spill file, since no version or entry then names a tuple + * id or a place past it and no insert has taken one; then it stores * the save's token in saved and returns the record a make would read back, * its counter the word's one load, which it keeps in the blob, so the blob * is the record the manifest holds once the save lands. A storage with no * base gives no record: idle, its state is the manifest's and turns clean, - * and otherwise its tuple ids are named only by versions its WAL redoes. + * and otherwise its tuple ids are named only by versions its WAL redoes. With + * a record, it gives the state's base end, read under the same lock. */ static bool txn_state_gathered(const HostSharedHeld *held, HostIoIo *io, uint64 token, bool *some, RepoGathered *result, - HostError *error) + HostLsn *end, HostError *error) { txn_state_gathered_frame_0_t frame = {.slot = {.io = io, .held = held, .error = error}, .token = token, - .some = some, .result = result}; + .some = some, .result = result, .end = end}; std_landing(&frame, txn_state_gathered_body_0, txn_state_gathered_pad_0); return frame.slot.ok; @@ -4262,6 +4550,7 @@ txn_state_gather_body_0(void *frame_) txn_state_gather_frame_0_t *frame = frame_; HostSharedRecord makers; uint64 token; + HostLsn latest = 0; if (!host_shared_single(&TXN_STATE_MAKERS_SINGLE_OPS, &makers, frame->error)) return; @@ -4280,14 +4569,16 @@ txn_state_gather_body_0(void *frame_) { const txn_TupleU64HeldState_t *state = &STD_VEC_AT(&frame->states, txn_TupleU64HeldState_t, k); bool some; + HostLsn end; - if (!txn_state_gathered(&state->f1, frame->io, token, &some, &frame->edit, frame->error)) + if (!txn_state_gathered(&state->f1, frame->io, token, &some, &frame->edit, &end, frame->error)) { txn_state_gather_pad_0(frame); return; } if (!some) continue; + latest = Max(latest, end); frame->edit_live = true; std_vec_push_Checked(&frame->edits, &frame->edit, sizeof(frame->edit), _Alignof(RepoGathered)); frame->edit_live = false; @@ -4298,6 +4589,7 @@ txn_state_gather_body_0(void *frame_) frame->locations_live = false; frame->result_1->token = token; frame->result_1->locations = frame->locations; + frame->result_1->end = latest; frame->ok = true; txn_state_gather_pad_0(frame); } @@ -4308,7 +4600,8 @@ txn_state_gather_body_0(void *frame_) * database whose saved isn't clean, read one at a time under each state's * lock, it gives the record a make would read back and stores the save's * token in saved, which saved turns to clean once the manifest that holds - * the record is written. + * the record is written. The saving's end is the latest base end among the + * records it gives. */ bool txn_state_gather(HostIoIo *io, uint32 database, const RepoRead *read, StdVec *result_0, TxnStateSaving *result_1, @@ -4877,6 +5170,7 @@ txn_state_held_to_end(HostIoIo *io, const HostStorage *storage, bool *some, TxnS #ifdef DOLTGRES_TEST_TXN #include "doltgres/chunks/trailer.h" #include "doltgres/simc/simc.h" +#include "doltgres/txn/memtable/testing.h" #include "doltgres/std/test.h" /* txn_state_tests_read_back asserts that a blob unkept reads back as a base record or none, as unwrap and assert_eq do at their places. */ @@ -5156,7 +5450,7 @@ test_txn_state_tests_a_state_is_filled_from_its_base_and_found_again(void) } /* txn_state_tests_reopened returns a process's own handle on a database's repository. */ -static bool +bool txn_state_tests_reopened(HostIoIo *io, uint32 database, RepoRepo *result, HostError *error) { ChunksErrorError failed; @@ -5718,7 +6012,7 @@ txn_state_tests_saved_closure_1(void *captures, HostIoIo *io, ChunksErrorError * } /* txn_state_tests_saved saves a database's repository through the gathering, running between after the manifest's write landed and before saved. */ -static void +void txn_state_tests_saved(HostIoIo *io, const RepoRepo *repo, uint32 database, void (*between) (void *captures, HostIoIo *io), void *between_captures) { @@ -6334,6 +6628,54 @@ test_txn_state_tests_a_linked_free_space_map_or_tuple_id_table_keeps_the_counter simc_TestIo_drop(&test_io); } +/* + * The fall-back words and spills: a state whose rows', free space map's, and + * tuple id table's words are clear but whose spills are linked, or one of + * whose spills is pending, keeps its counter through a save, since a spill + * holds versions of rows and takes of places past its base's; it falls back + * once both are clear. + */ +void +test_txn_state_tests_a_state_with_spills_keeps_the_counter_from_falling_back(void) +{ + static const size_t words[] = {TXN_STATE_SPILLS_INDEX, TXN_STATE_SPILLS_PENDING}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostStorage s = txn_state_tests_storage(9216, 16384); + RepoRepo repo = txn_state_tests_created(io, 9216); + TxnBaseBase put = txn_base_tests_full(); + HostSharedReader reading; + HostError error; + HostSharedHeld held; + HostSharedRecord record; + + put.counter = 5; + put.fresh_end = 3; + txn_state_tests_put(io, &repo, &s, &put); + if (!host_shared_read(io, &reading, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + held = txn_state_tests_made(__LINE__, 9, io, &repo, &reading, &s); + record = host_shared_Held_record(&held); + for (size_t k = 0; k < lengthof(words); k++) + { + txn_state_exchange(record, TXN_STATE_COUNTER, txn_state_load(record, TXN_STATE_COUNTER), 2 << 16 | 4); + txn_state_exchange(record, TXN_STATE_SAVED, TXN_STATE_CLEAN, 0); + txn_state_exchange(record, words[k], 0, 1); + txn_state_tests_saved(io, &repo, 9216, txn_state_tests_nothing, NULL); + txn_state_tests_counter_is(__LINE__, 9, record, 2 << 16 | 4, k); + txn_state_exchange(record, words[k], 1, 0); + txn_state_exchange(record, TXN_STATE_SAVED, TXN_STATE_CLEAN, 0); + txn_state_tests_saved(io, &repo, 9216, txn_state_tests_nothing, NULL); + txn_state_tests_counter_is(__LINE__, 9, record, 3, k); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + txn_base_Base_drop(&put); + simc_TestIo_drop(&test_io); +} + /* txn_state_tests_column is the key case's key column of an attribute number. */ static TxnBaseKeyColumn txn_state_tests_column(uint16 attnum) @@ -6460,6 +6802,7 @@ txn_state_tests_folded(uint64 folded) TxnBaseBase base = txn_base_tests_full(); base.folded = folded; + base.read = folded; return base; } @@ -6493,7 +6836,7 @@ txn_state_tests_installs(uint32 line, uint32 col, HostIoIo *io, const HostShared bool installed; HostError error; - if (!txn_state_installed(HOST_IO_IO_TAKES_SHORT(io), held, read, base, &installed, &error)) + if (!txn_state_installed(HOST_IO_IO_TAKES_SHORT(io), held, read, base, NULL, NULL, &installed, &error)) STD_RESULT_EXPECT_FAILED(line, col, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); return installed; @@ -6596,7 +6939,7 @@ test_txn_state_tests_an_install_keeps_a_key_recorded_after_its_fold_read_the_bas host_Error_Debug_fmt); host_shared_Held_drop(&dropped); folding.folded = 10; - if (!txn_state_installed(HOST_IO_IO_TAKES_SHORT(io), &held, read, &folding, &installed, &error)) + if (!txn_state_installed(HOST_IO_IO_TAKES_SHORT(io), &held, read, &folding, NULL, NULL, &installed, &error)) STD_RESULT_EXPECT_FAILED(2307, 77, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); STD_ASSERT(2307, 9, installed, "installed(io, &held, read, &Base { folded: 10, ..folding }).unwrap()"); @@ -6737,4 +7080,160 @@ test_txn_state_tests_a_reclaim_keeps_a_state_whose_manifest_lists_a_spill(void) simc_TestIo_drop(&test_io); } +/* + * Bugs row 340's case (RevB 4's F1): a pool full of idle unsaved states, + * each holding a running writer's version and no base, which no save turns + * clean, gives a further make the pool's error: the reclaim's save makes no + * progress, so the make doesn't look again. + */ +void +test_txn_state_tests_a_make_while_running_writers_hold_every_unsaved_state_gets_the_pools_error(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo; + HostSharedReader reading; + HostError error; + HostXid sub; + HostXid top; + uint32 relation = 1; + + simc_end(false); + repo = txn_state_tests_created(io, 9211); + if (!host_shared_read(io, &reading, &error) || !host_current_writer(HOST_IO_IO_IO_DOORS(io), &sub, &top, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (;; relation++) + { + HostStorage s = txn_state_tests_storage(9211, relation); + HostSharedHeld held; + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "row", 3); + TxnMemtableVersionChange change = {.writer = txn_memtable_testing_writer(sub.f0), .op = &op, + .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + + if (!txn_state_state(io, &repo, &reading, &s, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &held, &error)) + { + txn_memtable_Op_drop(&op); + txn_state_tests_full_pool(&error); + host_Error_drop(&error); + break; + } + op.u.put.tuple.f0 = 1; + op.u.put.heap_bytes = 32; + if (!txn_memtable_table_push(io, host_shared_Held_record(&held), (const uint8 *) "k", 1, &change, &handle, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_Op_drop(&op); + txn_state_unsaved(host_shared_Held_record(&held)); + host_shared_Held_drop(&held); + } + STD_ASSERT(__LINE__, 0, relation > 1, "the pool held a state"); + simc_end(true); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + simc_TestIo_drop(&test_io); +} + + +/* txn_state_tests_keying_t is the key case's rechain logging's captures: what a backend's make records the key with. */ +typedef struct txn_state_tests_keying_t +{ + HostIoIo *io; + const RepoRepo *repo; + const HostSharedReader *reading; + const HostStorage *storage; + TxnStateMade *made; +} txn_state_tests_keying_t; + +/* txn_state_tests_keying is the key case's rechain logging: a backend's make records the state's key meanwhile. */ +static bool +txn_state_tests_keying(void *captures, HostIoIoDoors io, HostLsn reached, HostLsn *end, HostError *error) +{ + txn_state_tests_keying_t *keying = captures; + HostSharedHeld made; + + (void) io; + (void) reached; + if (!txn_state_state(keying->io, keying->repo, keying->reading, keying->storage, TXN_STATE_PERSISTENCE_PERMANENT, + true, keying->made, &made, error)) + return false; + host_shared_Held_drop(&made); + *end = 0; + return true; +} + +/* + * Bugs row 350's window (RevB 4's R1): a backend that records a storage's + * key while a rechain logs its record, with the state's lock released, keeps + * it across the rechain's replace, which takes the key of the newest base it + * replaces. + */ +void +test_txn_state_tests_a_rechain_keeps_a_key_recorded_while_it_logs(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + RepoRepo repo = txn_state_tests_created(io, 9403); + HostStorage s = txn_state_tests_storage(9403, 16384); + TxnBaseKey none = {.tag = TXN_BASE_KEY_NONE}; + TxnBaseKey key = {.tag = TXN_BASE_KEY_COLUMNS}; + TxnBaseKeyColumn column = {.attnum = 1, .class = 1978, .typid = 23}; + TxnBaseBase unknown = txn_base_Base_empty(&none, 16401, false, 0); + TxnBaseBase chaining; + TxnBaseBase based; + TxnStateMade made = {.table = 16401}; + txn_state_tests_keying_t keying = {.io = io, .repo = &repo, .storage = &s, .made = &made}; + HostSharedReader reading; + HostError error; + HostSharedHeld held; + StdOptionRef left; + StdOptionRef right; + uint64 read; + bool some; + bool replaced; + + unknown.key.tag = TXN_BASE_KEY_UNKNOWN; + txn_state_tests_put(io, &repo, &s, &unknown); + if (!host_shared_read(io, &reading, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + keying.reading = &reading; + held = txn_state_tests_made(__LINE__, 0, io, &repo, &reading, &s); + read = txn_state_generation(host_shared_Held_record(&held)); + if (!txn_state_newest(io, &held, &some, &chaining, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (!some) + STD_PANIC(__LINE__, 0, "the state holds no base"); + std_vec_with_capacity_Checked(&key.u.columns, 1, sizeof(TxnBaseKeyColumn), _Alignof(TxnBaseKeyColumn)); + STD_VEC_AT(&key.u.columns, TxnBaseKeyColumn, 0) = column; + key.u.columns.len = 1; + txn_base_Key_Clone_clone(&key, &made.key); + if (!txn_state_rechained(HOST_IO_IO_TAKES_SHORT(io), &held, read, &chaining, txn_state_tests_keying, &keying, + &replaced, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, replaced, "the rechain replaced the newest base"); + txn_base_Base_drop(&chaining); + if (!txn_state_base_record(io, &held, &some, &based, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + left.has = some; + left.some = STD_ITEM(&based.key, txn_base_Key_Debug_fmt); + right.has = true; + right.some = STD_ITEM(&key, txn_base_Key_Debug_fmt); + STD_ASSERT_EQ(__LINE__, 0, some && txn_base_Key_PartialEq_eq(&based.key, &key), &left, + std_option_Debug_fmt, &right, std_option_Debug_fmt); + if (some) + txn_base_Base_drop(&based); + txn_base_Key_drop(&key); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + txn_base_Base_drop(&unknown); + simc_TestIo_drop(&test_io); +} + #endif diff --git a/src/doltgres/txn/c/status.c b/src/doltgres/txn/c/status.c index 2ca2c8516d462..ec096d87adf10 100644 --- a/src/doltgres/txn/c/status.c +++ b/src/doltgres/txn/c/status.c @@ -1040,13 +1040,14 @@ typedef struct txn_status_counted_frame_t HostIoTakesShort io; const HostDeltaHold *hold; TxnStatusRoomed roomed; + uint64 amount; HostError *error; bool ok; HostSharedGuard guard; bool guard_live; } txn_status_counted_frame_t; -/* txn_status_counted_body adds one to the entry's count under the table's lock shared. */ +/* txn_status_counted_body adds the amount to the entry's count under the table's lock shared. */ static void txn_status_counted_body(void *frame_) { @@ -1063,7 +1064,7 @@ txn_status_counted_body(void *frame_) if (!txn_status_slotted(frame->hold, root, frame->roomed.f0, &page, &at, frame->error) || !txn_status_word(&page, at + 2, &word, frame->error)) return; - host_delta_Word_fetch_add(&word, 1); + host_delta_Word_fetch_add(&word, frame->amount); frame->ok = true; } @@ -1079,15 +1080,16 @@ txn_status_counted_pad(void *frame_) } /* - * txn_status_counted adds one to the count of the entry a take's slot names, - * an atomic add under the table's lock shared with no lookup and no check of - * the slot, for a version's push inside its window under the hold the push - * holds. + * txn_status_counted adds an amount to the count of the entry a take's slot + * names, a subtract being the add of its two's complement, an atomic add + * under the table's lock shared with no lookup and no check of the slot, for + * a version's push inside its window under the hold the push holds. */ bool -txn_status_counted(HostIoTakesShort io, const HostDeltaHold *hold, TxnStatusRoomed roomed, HostError *error) +txn_status_counted(HostIoTakesShort io, const HostDeltaHold *hold, TxnStatusRoomed roomed, uint64 amount, + HostError *error) { - txn_status_counted_frame_t frame = {.io = io, .hold = hold, .roomed = roomed, .error = error}; + txn_status_counted_frame_t frame = {.io = io, .hold = hold, .roomed = roomed, .amount = amount, .error = error}; std_landing(&frame, txn_status_counted_body, txn_status_counted_pad); txn_status_counted_pad(&frame); @@ -1347,6 +1349,23 @@ txn_status_prepared_committing(HostIoIo *io, HostXid xid, HostError *error) return true; } +/* + * txn_status_kept_roomed returns the slot of the entry a version's writer's + * top-level xid has, taking no room for one with none, so a version carried + * or read back counts where its writer's other versions do. + */ +bool +txn_status_kept_roomed(HostIoIo *io, const TxnMemtableVersionWriter *writer, bool *has_result, + TxnStatusRoomed *result, HostError *error) +{ + HostXid top = {writer->has_top ? writer->top : writer->xid}; + TxnStatusEntry entry; + + if (!txn_status_lookup(io, top, has_result, &entry, error)) + return false; + return !*has_result || txn_status_redo_roomed(io, writer, has_result, result, error); +} + /* * txn_status_redo_roomed takes the room of a replayed row change's top-level * xid pending in no process, and returns its slot; a writer below the first @@ -2009,4 +2028,672 @@ test_txn_status_tests_a_key_whose_slot_isnt_stored_yet_leaves_its_slot_as_it_was simc_TestIo_drop(&test_io); } +/* + * The fork's own: each outcome's word says that outcome again, a zero word is + * pending in no process, and the kinds' words don't meet. + */ +void +test_txn_status_tests_an_outcomes_word_says_the_outcome(void) +{ + TxnStatusOutcome outcomes[6] = {{.tag = TXN_STATUS_OUTCOME_UNOWNED}, + {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = 1}, + {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = UINT64CONST(0x16B3F2A40)}, + {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = TXN_STATUS_LSNS - 1}, + {.tag = TXN_STATUS_OUTCOME_PENDING_IN, .u.pending_in = 0}, + {.tag = TXN_STATUS_OUTCOME_PENDING_IN, .u.pending_in = PG_UINT32_MAX}}; + TxnStatusOutcome unowned = {.tag = TXN_STATUS_OUTCOME_UNOWNED}; + TxnStatusOutcome zero = txn_status_Outcome_of(0); + uint64 words[6]; + + for (size_t k = 0; k < lengthof(outcomes); k++) + { + TxnStatusOutcome again; + + STD_ASSERT(__LINE__, 0, txn_status_Outcome_word(outcomes[k], &words[k]), "the outcome has a word"); + again = txn_status_Outcome_of(words[k]); + STD_ASSERT_EQ(__LINE__, 0, txn_status_Outcome_PartialEq_eq(&again, &outcomes[k]), &again, + txn_status_Outcome_Debug_fmt, &outcomes[k], txn_status_Outcome_Debug_fmt); + for (size_t j = 0; j < k; j++) + STD_ASSERT(__LINE__, 0, words[j] != words[k], "!words.contains(&word)"); + } + STD_ASSERT_EQ(__LINE__, 0, txn_status_Outcome_PartialEq_eq(&zero, &unowned), &zero, txn_status_Outcome_Debug_fmt, + &unowned, txn_status_Outcome_Debug_fmt); +} + +/* The fork's own: a commit record's end of zero or past the word's room has no word. */ +void +test_txn_status_tests_an_outcome_too_large_for_its_word_has_none(void) +{ + TxnStatusOutcome at_zero = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = 0}; + TxnStatusOutcome past = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = TXN_STATUS_LSNS}; + uint64 word; + + STD_ASSERT(__LINE__, 0, !txn_status_Outcome_word(at_zero, &word), "Outcome::Committed(0).word() == None"); + STD_ASSERT(__LINE__, 0, !txn_status_Outcome_word(past, &word), "Outcome::Committed(LSNS).word() == None"); +} + +/* + * The fork's own: every normal xid but the last has its own xid word, which + * is neither a free place's nor a removed one's, and the invalid, bootstrap, + * and frozen xids have none. + */ +void +test_txn_status_tests_an_xids_word_is_no_empty_places(void) +{ + static const uint64 none[4] = {0, 1, 2, PG_UINT64_MAX}; + static const uint64 some[4] = {3, 747, PG_UINT32_MAX, PG_UINT64_MAX - 1}; + uint64 key; + + for (size_t k = 0; k < lengthof(none); k++) + { + HostXid xid = {none[k]}; + + STD_ASSERT(__LINE__, 0, !txn_status_key(xid, &key), "key(Xid(xid)) == None"); + } + for (size_t k = 0; k < lengthof(some); k++) + { + HostXid xid = {some[k]}; + + STD_ASSERT(__LINE__, 0, txn_status_key(xid, &key) && key == some[k], "key(Xid(xid)) == Some(xid)"); + } +} + +/* + * The fork's own: a lookup's places are within the table and reach every one + * of them, and xids taken in order start at places spread over it. + */ +void +test_txn_status_tests_a_lookup_probes_every_place_and_neighbours_start_apart(void) +{ + static const uint64 capacities[4] = {1, 2, 64, 4096}; + bool seen[4096]; + size_t distinct = 0; + + for (size_t c = 0; c < lengthof(capacities); c++) + { + uint64 capacity = capacities[c]; + uint64 start = txn_status_first(747, capacity); + uint64 place = start; + + memset(seen, 0, sizeof(seen)); + for (uint64 k = 0; k < capacity; k++) + { + STD_ASSERT(__LINE__, 0, place < capacity && !seen[place], "place < capacity && !seen[place]"); + seen[place] = true; + place = txn_status_next(place, capacity); + } + STD_ASSERT_EQ_U64(__LINE__, 0, place, start); + } + memset(seen, 0, sizeof(seen)); + for (uint64 xid = 1000; xid < 2000; xid++) + { + uint64 start = txn_status_first(xid, 4096); + + distinct += !seen[start]; + seen[start] = true; + } + STD_ASSERT(__LINE__, 0, distinct > 850, "more than 850 of 1000 neighbours start at their own place"); +} + +/* The fork's own: a table is crowded from three quarters of its places taken. */ +void +test_txn_status_tests_a_table_is_crowded_at_three_quarters(void) +{ + STD_ASSERT(__LINE__, 0, + !txn_status_crowded(0, 4) && !txn_status_crowded(2, 4) && txn_status_crowded(3, 4) && + txn_status_crowded(4, 4), "crowded at three of four"); + STD_ASSERT(__LINE__, 0, !txn_status_crowded(3071, 4096) && txn_status_crowded(3072, 4096), + "crowded at 3072 of 4096"); +} + +/* txn_status_tests_capacity returns the table's index capacity, read from its root. */ +static uint64 +txn_status_tests_capacity(HostIoIo *io) +{ + TxnStatusScope scope = {.io = io}; + HostError error; + uint64 capacity; + + scope.error = &error; + if (!txn_status_hold(&scope) || !txn_status_root(&scope)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + capacity = txn_status_load(scope.root, TXN_STATUS_CAPACITY); + txn_status_release(&scope); + return capacity; +} + +/* txn_status_tests_took takes an xid's entry with an outcome and returns whether the entry was there already. */ +static bool +txn_status_tests_took(HostIoIo *io, HostXid xid, TxnStatusOutcome outcome) +{ + TxnStatusRoomed roomed; + HostError error; + bool had; + + if (!txn_status_take(io, xid, outcome, &roomed, &had, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return had; +} + +/* txn_status_tests_added adds an amount to an xid's entry's count and returns whether it has an entry, with the count in a result. */ +static bool +txn_status_tests_added(HostIoIo *io, HostXid xid, uint64 amount, uint64 *result) +{ + HostError error; + bool some; + + if (!txn_status_add(io, xid, amount, &some, result, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return some; +} + +/* txn_status_tests_looked_up looks up an xid's entry and returns whether it has one, the entry in a result. */ +static bool +txn_status_tests_looked_up(HostIoIo *io, HostXid xid, TxnStatusEntry *result) +{ + HostError error; + bool some; + + if (!txn_status_lookup(io, xid, &some, result, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return some; +} + +/* txn_status_tests_removed removes an xid's entry and returns whether it had one. */ +static bool +txn_status_tests_removed(HostIoIo *io, HostXid xid) +{ + HostError error; + bool removed; + + if (!txn_status_remove(io, xid, &removed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return removed; +} + +/* + * The fork's own (design item 7): a take of room makes an entry a lookup + * finds, pending, with a count of zero, a second take for its xid leaves it, + * its count adds and subtracts, a removal leaves it for no lookup, and the + * table grows past its load factor keeping every entry; a growth leaves + * behind an entry of zero count whose xid isn't running. + */ +void +test_txn_status_tests_entries_are_taken_found_counted_removed_and_kept_by_a_growth(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + TxnStatusOutcome unowned = {.tag = TXN_STATUS_OUTCOME_UNOWNED}; + TxnStatusOutcome pending = {.tag = TXN_STATUS_OUTCOME_PENDING_IN, .u.pending_in = 3}; + TxnStatusEntry wanted = {.outcome = pending}; + HostXid seven = {7}; + HostXid eight = {8}; + HostXid ended; + HostXid aborted; + HostXid running; + HostXid committed[TXN_STATUS_PLACES]; + TxnStatusEntry entry; + uint64 count; + + STD_ASSERT(__LINE__, 0, !txn_status_tests_looked_up(io, seven, &entry), "lookup(io, Xid(7)).unwrap() == None"); + STD_ASSERT(__LINE__, 0, !txn_status_tests_took(io, seven, pending), "!take(io, Xid(7), PendingIn(3)).1"); + STD_ASSERT_EQ(__LINE__, 0, txn_status_tests_looked_up(io, seven, &entry) && + txn_status_Entry_PartialEq_eq(&entry, &wanted), &entry, txn_status_Entry_Debug_fmt, &wanted, + txn_status_Entry_Debug_fmt); + STD_ASSERT(__LINE__, 0, txn_status_tests_took(io, seven, unowned), "take(io, Xid(7), Unowned).1"); + STD_ASSERT(__LINE__, 0, txn_status_tests_added(io, seven, 2, &count) && count == 2, + "add(io, Xid(7), 2) == Some(2)"); + STD_ASSERT(__LINE__, 0, txn_status_tests_added(io, seven, PG_UINT64_MAX, &count) && count == 1, + "add(io, Xid(7), -1) == Some(1)"); + STD_ASSERT(__LINE__, 0, !txn_status_tests_added(io, eight, 1, &count), "add(io, Xid(8), 1) == None"); + STD_ASSERT(__LINE__, 0, txn_status_tests_removed(io, seven) && !txn_status_tests_removed(io, seven), + "remove(io, Xid(7)) && !remove(io, Xid(7))"); + STD_ASSERT(__LINE__, 0, !txn_status_tests_looked_up(io, seven, &entry), "lookup(io, Xid(7)).unwrap() == None"); + ended = simc_begin(); + aborted = simc_begin(); + running = simc_begin(); + simc_commit(ended); + simc_abort(aborted); + txn_status_tests_took(io, ended, unowned); + txn_status_tests_took(io, aborted, unowned); + txn_status_tests_took(io, running, unowned); + txn_status_tests_added(io, aborted, 1, &count); + for (uint64 k = 0; k < TXN_STATUS_PLACES; k++) + committed[k] = simc_begin(); + for (uint64 k = 0; k < TXN_STATUS_PLACES; k++) + { + simc_commit(committed[k]); + txn_status_tests_took(io, committed[k], unowned); + txn_status_tests_added(io, committed[k], 1, &count); + } + STD_ASSERT(__LINE__, 0, txn_status_tests_capacity(io) >= 2 * TXN_STATUS_PLACES, + "root.atomic::(Op::Load) >= 2 * PLACES"); + for (uint64 k = 0; k < TXN_STATUS_PLACES; k++) + STD_ASSERT(__LINE__, 0, txn_status_tests_looked_up(io, committed[k], &entry) && entry.count == 1, + "a committed xid's entry keeps its count of 1"); + STD_ASSERT(__LINE__, 0, !txn_status_tests_looked_up(io, ended, &entry), "lookup(io, ended).unwrap() == None"); + STD_ASSERT(__LINE__, 0, !txn_status_tests_looked_up(io, aborted, &entry), "lookup(io, aborted).unwrap() == None"); + STD_ASSERT(__LINE__, 0, txn_status_tests_looked_up(io, running, &entry) && entry.count == 0, + "lookup(io, running).unwrap().map(|e| e.count) == Some(0)"); + simc_TestIo_drop(&test_io); +} + +/* + * The fork's own (PRevPF 5's F1): a growth is sized from the entries it + * keeps, so churn alone keeps one segment: 40,000 transactions, each taking + * room, counting, committing, and counting down, every other one removed and + * the rest left at a count of zero, grow the table many times and leave it at + * its first capacity. + */ +void +test_txn_status_tests_churn_with_no_live_entries_keeps_one_segment(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + TxnStatusOutcome pending = {.tag = TXN_STATUS_OUTCOME_PENDING_IN, .u.pending_in = 1}; + uint64 count; + + for (uint32 at = 0; at < 40000; at++) + { + HostXid xid = simc_begin(); + + txn_status_tests_took(io, xid, pending); + txn_status_tests_added(io, xid, 1, &count); + simc_commit(xid); + txn_status_tests_added(io, xid, PG_UINT64_MAX, &count); + if (at % 2 == 0) + txn_status_tests_removed(io, xid); + } + STD_ASSERT_EQ_U64(__LINE__, 0, txn_status_tests_capacity(io), TXN_STATUS_PLACES); + simc_TestIo_drop(&test_io); +} + +/* + * The fork's own: a growth's index capacity is the fewest whole segments of + * places that hold what it keeps under the load factor. + */ +void +test_txn_status_tests_a_growth_holds_what_it_keeps_in_the_least_room(void) +{ + STD_ASSERT_EQ_U64(__LINE__, 0, txn_status_sized(0), TXN_STATUS_PLACES); + STD_ASSERT_EQ_U64(__LINE__, 0, txn_status_sized(TXN_STATUS_PLACES * 3 / 4 - 1), TXN_STATUS_PLACES); + STD_ASSERT_EQ_U64(__LINE__, 0, txn_status_sized(TXN_STATUS_PLACES * 3 / 4), 2 * TXN_STATUS_PLACES); + STD_ASSERT_EQ_U64(__LINE__, 0, txn_status_sized(2 * TXN_STATUS_PLACES * 3 / 4 - 1), 2 * TXN_STATUS_PLACES); + STD_ASSERT_EQ_U64(__LINE__, 0, txn_status_sized(2 * TXN_STATUS_PLACES * 3 / 4), 3 * TXN_STATUS_PLACES); +} + +/* TXN_STATUS_TESTS_LISTED is LISTED, the commits' slots at their position in the test's table. */ +static const HostSharedListed TXN_STATUS_TESTS_LISTED = {0}; + +/* txn_status_tests_roomed takes a top-level xid's room as a backend's first write does. */ +static void +txn_status_tests_roomed(HostIoIo *io, HostXid top) +{ + TxnStatusRoomed roomed; + HostError error; + + if (!txn_status_roomed(io, top, &roomed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); +} + +/* txn_status_tests_commit returns what the table says of a top-level xid's commit. */ +static TxnStatusCommit +txn_status_tests_commit(HostIoIo *io, HostXid xid) +{ + TxnStatusCommit commit; + HostError error; + + if (!txn_status_committed(io, xid, &commit, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return commit; +} + +/* txn_status_tests_commit_is asserts at its line that the table says a commit of an xid. */ +static void +txn_status_tests_commit_is(uint32 line, HostIoIo *io, HostXid xid, TxnStatusCommit wanted) +{ + TxnStatusCommit commit = txn_status_tests_commit(io, xid); + + STD_ASSERT_EQ(line, 0, txn_status_Commit_PartialEq_eq(&commit, &wanted), &commit, txn_status_Commit_Debug_fmt, + &wanted, txn_status_Commit_Debug_fmt); +} + +/* txn_status_tests_mine returns this process's slot of the commits' words. */ +static HostSharedMine +txn_status_tests_mine(HostIoIo *io) +{ + HostSharedMine mine; + HostError error; + + if (!host_shared_mine(HOST_IO_IO_TAKES_SHORT(io), &TXN_STATUS_COMMITS_PROC_SLOTS_OPS, &mine, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return mine; +} + +/* txn_status_tests_moved moves this process's slot's pair into its entry. */ +static void +txn_status_tests_moved(HostIoIo *io, const HostSharedMine *mine) +{ + HostError error; + + if (!txn_status_moved(io, mine, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); +} + +/* + * PLAN item 7's registration, move, and reader: an entry pending in this + * process reads not committed while its transaction runs; once the + * transaction commits with its pair registered, a reader takes the end from + * the slot; the move stores the end in the entry and clears the slot's xid, + * and a second move does nothing; an entry nothing names is dropped by its + * move; and an xid with no entry reads missing. + */ +void +test_txn_status_tests_a_commit_registered_in_its_slot_reads_back_and_its_move_settles_the_entry(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedMine mine = txn_status_tests_mine(io); + HostXid named = simc_begin(); + HostXid unnamed = simc_begin(); + HostXid absent = simc_begin(); + TxnStatusCommit pending = {.tag = TXN_STATUS_COMMIT_PENDING}; + TxnStatusCommit at = {.tag = TXN_STATUS_COMMIT_AT, .u.at = 400}; + TxnStatusCommit missing = {.tag = TXN_STATUS_COMMIT_MISSING}; + TxnStatusEntry settled = {.outcome = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = 400}, .count = 2}; + TxnStatusEntry entry; + uint64 count; + + txn_status_tests_roomed(io, named); + txn_status_tests_added(io, named, 2, &count); + txn_status_tests_commit_is(__LINE__, io, named, pending); + txn_status_registered(&TXN_STATUS_TESTS_LISTED, named, 400); + simc_commit(named); + txn_status_tests_commit_is(__LINE__, io, named, at); + txn_status_tests_moved(io, &mine); + STD_ASSERT_EQ_U64(__LINE__, 0, host_shared_Mine_load(&mine, TXN_STATUS_COMMIT_XID_INDEX), 0); + STD_ASSERT_EQ(__LINE__, 0, txn_status_tests_looked_up(io, named, &entry) && + txn_status_Entry_PartialEq_eq(&entry, &settled), &entry, txn_status_Entry_Debug_fmt, &settled, + txn_status_Entry_Debug_fmt); + txn_status_tests_moved(io, &mine); + txn_status_tests_commit_is(__LINE__, io, named, at); + txn_status_tests_roomed(io, unnamed); + txn_status_registered(&TXN_STATUS_TESTS_LISTED, unnamed, 500); + simc_commit(unnamed); + txn_status_tests_moved(io, &mine); + STD_ASSERT(__LINE__, 0, !txn_status_tests_looked_up(io, unnamed, &entry), "lookup(io, unnamed).unwrap() == None"); + txn_status_tests_commit_is(__LINE__, io, absent, missing); + simc_TestIo_drop(&test_io); +} + +/* + * The fold text's 3.1 (draft 10): a fold's point lies below the end of a + * commit registered at or below the insert position whose transaction is + * still in the proc array, and is the position once it left, or when the + * commit ends past the position. + */ +void +test_txn_status_tests_a_folds_point_lies_below_a_commit_still_in_the_proc_array(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedMine mine = txn_status_tests_mine(io); + HostXid held = simc_begin(); + HostError error; + HostLsn point; + uint64 count; + + txn_status_tests_roomed(io, held); + txn_status_tests_added(io, held, 1, &count); + txn_status_registered(&TXN_STATUS_TESTS_LISTED, held, 400); + if (!txn_status_lowered(io, 500, &point, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ_U64(__LINE__, 0, point, 399); + if (!txn_status_lowered(io, 300, &point, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ_U64(__LINE__, 0, point, 300); + simc_commit(held); + if (!txn_status_lowered(io, 500, &point, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ_U64(__LINE__, 0, point, 500); + txn_status_tests_moved(io, &mine); + simc_TestIo_drop(&test_io); +} + +/* + * PLAN item 7's take of room: a backend's first write in a transaction moves + * its slot's pair, here a committed earlier transaction's, and takes its + * top-level xid's room pending in this process; a second write takes + * nothing. + */ +void +test_txn_status_tests_a_take_of_room_moves_the_slots_pair_first(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + TxnStatusEntry earlier_entry = {.outcome = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = 700}, + .count = 1}; + TxnStatusEntry later_entry = {.outcome = {.tag = TXN_STATUS_OUTCOME_PENDING_IN, + .u.pending_in = host_shared_proc_number()}}; + HostXid earlier = simc_begin(); + HostXid later = simc_begin(); + TxnStatusEntry entry; + uint64 count; + + txn_status_tests_roomed(io, earlier); + txn_status_tests_added(io, earlier, 1, &count); + txn_status_registered(&TXN_STATUS_TESTS_LISTED, earlier, 700); + simc_commit(earlier); + txn_status_tests_roomed(io, later); + STD_ASSERT_EQ(__LINE__, 0, txn_status_tests_looked_up(io, earlier, &entry) && + txn_status_Entry_PartialEq_eq(&entry, &earlier_entry), &entry, txn_status_Entry_Debug_fmt, + &earlier_entry, txn_status_Entry_Debug_fmt); + STD_ASSERT_EQ(__LINE__, 0, txn_status_tests_looked_up(io, later, &entry) && + txn_status_Entry_PartialEq_eq(&entry, &later_entry), &entry, txn_status_Entry_Debug_fmt, + &later_entry, txn_status_Entry_Debug_fmt); + txn_status_tests_added(io, later, 1, &count); + txn_status_tests_roomed(io, later); + STD_ASSERT(__LINE__, 0, txn_status_tests_looked_up(io, later, &entry) && entry.count == 1, + "lookup(io, later).unwrap().map(|e| e.count) == Some(1)"); + simc_TestIo_drop(&test_io); +} + +/* + * PLAN item 7's redo call site: a replayed commit stores its record's end in + * the entry its replayed row change took, pending in no process, and a commit + * of a transaction with no entry stores nothing. + */ +void +test_txn_status_tests_a_replayed_commit_stores_its_end_in_the_entry_its_replay_took(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + TxnStatusOutcome at = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = 900}; + HostXid wrote = simc_begin(); + HostXid quiet = simc_begin(); + HostXid sub = simc_begin_under(wrote); + TxnMemtableVersionWriter writer = {.xid = sub.f0, .has_top = true, .top = wrote.f0}; + TxnStatusRoomed roomed; + TxnStatusEntry entry; + HostError error; + bool some; + + if (!txn_status_redo_roomed(io, &writer, &some, &roomed, &error) || + !txn_status_redo_committed(io, wrote, 900, &error) || !txn_status_redo_committed(io, quiet, 950, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ(__LINE__, 0, txn_status_tests_looked_up(io, wrote, &entry) && + txn_status_Outcome_PartialEq_eq(&entry.outcome, &at), &entry.outcome, + txn_status_Outcome_Debug_fmt, &at, txn_status_Outcome_Debug_fmt); + STD_ASSERT(__LINE__, 0, !txn_status_tests_looked_up(io, quiet, &entry), "lookup(io, quiet).unwrap() == None"); + simc_TestIo_drop(&test_io); +} + +/* + * PLAN item 7's prepared transaction: its entry, left pending in its + * preparing backend's number, is pointed at the committing process ahead of + * the critical section, whose registration then stores its end, which a + * reader takes from the committer's slot; a prepared transaction with no + * entry registers nothing. + */ +void +test_txn_status_tests_a_prepared_commit_points_its_entry_at_the_committer_and_registers(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + TxnStatusOutcome preparer = {.tag = TXN_STATUS_OUTCOME_PENDING_IN, .u.pending_in = 77}; + TxnStatusEntry pointed = {.outcome = {.tag = TXN_STATUS_OUTCOME_PENDING_IN, + .u.pending_in = host_shared_proc_number()}, .count = 1}; + TxnStatusCommit at = {.tag = TXN_STATUS_COMMIT_AT, .u.at = 1200}; + TxnStatusCommit missing = {.tag = TXN_STATUS_COMMIT_MISSING}; + HostXid prepared = simc_begin(); + HostXid empty = simc_begin(); + TxnStatusEntry entry; + HostError error; + uint64 count; + + txn_status_tests_took(io, prepared, preparer); + txn_status_tests_added(io, prepared, 1, &count); + if (!txn_status_prepared_committing(io, prepared, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ(__LINE__, 0, txn_status_tests_looked_up(io, prepared, &entry) && + txn_status_Entry_PartialEq_eq(&entry, &pointed), &entry, txn_status_Entry_Debug_fmt, &pointed, + txn_status_Entry_Debug_fmt); + txn_status_prepared_registered(&TXN_STATUS_TESTS_LISTED, 1200); + simc_commit(prepared); + txn_status_tests_commit_is(__LINE__, io, prepared, at); + if (!txn_status_prepared_committing(io, empty, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_status_prepared_registered(&TXN_STATUS_TESTS_LISTED, 1300); + simc_commit(empty); + txn_status_tests_commit_is(__LINE__, io, prepared, at); + txn_status_tests_commit_is(__LINE__, io, empty, missing); + simc_TestIo_drop(&test_io); +} + +/* + * The fold text's 3.11: a growth that rebuilds the index between a take of + * room and a count leaves the entry in its slot, so the count lands in the + * same entry, and a removed entry's slot is the next take's. + */ +void +test_txn_status_tests_a_slot_stays_its_entrys_across_a_growth_and_a_removed_ones_is_taken_again(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + TxnStatusOutcome unowned = {.tag = TXN_STATUS_OUTCOME_UNOWNED}; + HostXid counted_one = simc_begin(); + HostXid gone; + HostXid next; + TxnStatusRoomed roomed; + TxnStatusRoomed freed_slot; + TxnStatusRoomed again; + TxnStatusEntry entry; + HostDeltaHold hold; + HostError error; + uint64 capacity; + uint64 count; + bool had; + + if (!txn_status_take(io, counted_one, unowned, &roomed, &had, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + capacity = txn_status_tests_capacity(io); + while (txn_status_tests_capacity(io) == capacity) + { + HostXid xid = simc_begin(); + + txn_status_tests_took(io, xid, unowned); + txn_status_tests_added(io, xid, 1, &count); + } + if (!host_delta_hold(io, &hold, &error) || + !txn_status_counted(HOST_IO_IO_TAKES_SHORT(io), &hold, roomed, 1, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + host_delta_Hold_drop(&hold); + STD_ASSERT(__LINE__, 0, txn_status_tests_looked_up(io, counted_one, &entry) && entry.count == 1, + "lookup(io, counted_one).unwrap().map(|e| e.count) == Some(1)"); + gone = simc_begin(); + next = simc_begin(); + if (!txn_status_take(io, gone, unowned, &freed_slot, &had, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 0, txn_status_tests_removed(io, gone), "remove(io, gone).unwrap()"); + if (!txn_status_take(io, next, unowned, &again, &had, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ(__LINE__, 0, txn_status_Roomed_PartialEq_eq(&again, &freed_slot) && !had, &again, + txn_status_Roomed_Debug_fmt, &freed_slot, txn_status_Roomed_Debug_fmt); + simc_TestIo_drop(&test_io); +} + +/* txn_status_tests_grown takes committed xids' room, each counted once, until the table grows. */ +static void +txn_status_tests_grown(HostIoIo *io) +{ + TxnStatusOutcome unowned = {.tag = TXN_STATUS_OUTCOME_UNOWNED}; + uint64 capacity = txn_status_tests_capacity(io); + uint64 count; + + while (txn_status_tests_capacity(io) == capacity) + { + HostXid xid = simc_begin(); + + simc_commit(xid); + txn_status_tests_took(io, xid, unowned); + txn_status_tests_added(io, xid, 1, &count); + } +} + +/* + * Bugs row 90: while the proc array doesn't answer for every running and + * prepared transaction, as in crash recovery before its end puts the prepared + * ones in it, a growth keeps an entry of zero count whose xid isn't in + * progress, which a growth with the proc array answering leaves behind; one + * with a count whose xid aborted goes either way, since the commit log + * answers it. + */ +void +test_txn_status_tests_a_growth_while_the_proc_array_cant_answer_keeps_an_ended_entry(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + TxnStatusOutcome unowned = {.tag = TXN_STATUS_OUTCOME_UNOWNED}; + TxnStatusEntry kept = {.outcome = unowned}; + HostXid ended = simc_begin(); + HostXid aborted = simc_begin(); + TxnStatusEntry entry; + uint64 count; + + simc_commit(ended); + simc_abort(aborted); + txn_status_tests_took(io, ended, unowned); + txn_status_tests_took(io, aborted, unowned); + txn_status_tests_added(io, aborted, 1, &count); + simc_unanswered(true); + txn_status_tests_grown(io); + simc_unanswered(false); + STD_ASSERT_EQ(__LINE__, 0, txn_status_tests_looked_up(io, ended, &entry) && + txn_status_Entry_PartialEq_eq(&entry, &kept), &entry, txn_status_Entry_Debug_fmt, &kept, + txn_status_Entry_Debug_fmt); + STD_ASSERT(__LINE__, 0, !txn_status_tests_looked_up(io, aborted, &entry), "lookup(io, aborted).unwrap() == None"); + txn_status_tests_grown(io); + STD_ASSERT(__LINE__, 0, !txn_status_tests_looked_up(io, ended, &entry), "lookup(io, ended).unwrap() == None"); + simc_TestIo_drop(&test_io); +} + #endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/vacuum.c b/src/doltgres/txn/c/vacuum.c index 69be525692ec7..6aa2cb0d20dd7 100644 --- a/src/doltgres/txn/c/vacuum.c +++ b/src/doltgres/txn/c/vacuum.c @@ -31,9 +31,11 @@ #include "doltgres/std/unwind.h" #include "doltgres/trees/cursor.h" #include "doltgres/trees/mutate.h" +#include "doltgres/txn/fold.h" #include "doltgres/txn/free_space.h" #include "doltgres/txn/memtable/table.h" #include "doltgres/txn/memtable/tuples.h" +#include "doltgres/txn/spill.h" #include "doltgres/txn/state.h" #include "doltgres/txn/vacuum.h" #include "doltgres/txn/values.h" @@ -174,6 +176,31 @@ txn_vacuum_below(HostIoIo *io, TreesStoreNodeStore store, const TxnBaseBase *bas return false; } +/* + * txn_vacuum_spilled_below replaces a block's entry below the memtable by its + * newest version in the storage's spills when they hold one, which a step on + * a block whose memtable holds no version starts from, as an insert's take + * does; a spilled removal leaves none. + */ +static bool +txn_vacuum_spilled_below(HostIoIo *io, HostSharedRecord record, uint32 block, bool *has_below, StdVec *below, + HostError *error) +{ + TxnMemtableMapEntry entry; + bool some; + + if (!txn_memtable_map_spilled_newest(io, record, block, &some, &entry, error)) + return false; + if (!some) + return true; + if (*has_below) + std_vec_free_Checked(below, 1, 1); + *has_below = entry.has_value; + if (entry.has_value) + *below = entry.value; + return true; +} + /* * txn_vacuum_first returns a block's free bytes before its free space * entry's oldest version in the delta: the base's entry's, else an untouched @@ -512,6 +539,193 @@ txn_vacuum_taken(const StdVec *versions, bool has_below, const StdVec *below, ui return frame.ok; } +/* txn_vacuum_spilled reports whether a block's spilled takes took a tuple id, which VACUUM frees only once a fold took their file. */ +static bool +txn_vacuum_spilled(const StdVec *spilled, uint64 tid) +{ + for (size_t i = 0; i < spilled->len; i++) + if (STD_VEC_AT(spilled, HostRow, i).f0 == tid) + return true; + return false; +} + +/* + * txn_vacuum_committed_ids moves out a take's value ids when the xid whose + * commit decides them committed, and gives none otherwise, since an aborted + * row's values went at its abort (bugs row 146). + */ +static bool +txn_vacuum_committed_ids(HostIoIo *io, TxnVacuumTook *take, StdVec *result, HostError *error) +{ + HostStatus status = HOST_STATUS_ABORTED; + + if (take->ids.len > 0) + { + HostXid named = {take->named}; + + if (!host_status(HOST_IO_IO_IO_DOORS(io), named, &status, error)) + return false; + } + if (status == HOST_STATUS_COMMITTED) + { + *result = take->ids; + std_vec_new(&take->ids, TXN_VACUUM_ID); + } + else + std_vec_new(result, TXN_VACUUM_ID); + return true; +} + +/* + * txn_vacuum_stored returns a found tuple id's stored ender from its ender's + * xid and its free's horizon, which is 0 only for a row its own writer ended + * (OWN) or for none: the xid marked OWN, the xid, or 0. + */ +static uint64 +txn_vacuum_stored(uint64 ender, uint64 horizon) +{ + return ender != 0 && horizon == 0 ? ender | TXN_BASE_OWN : ender; +} + +/* txn_vacuum_logged returns a pending tuple id's free's conflict horizon: its ender's xid, or 0 for a row its own writer ended (OWN). */ +static uint64 +txn_vacuum_logged(uint64 ender) +{ + return (ender & TXN_BASE_OWN) == 0 ? ender : 0; +} + +/* txn_vacuum_leaving_frame_t is leaving's state across its pad: the run's takers, the base's entry below, and the taken offsets, while live. */ +typedef struct txn_vacuum_leaving_frame_t +{ + HostIoIo *io; + TreesStoreNodeStore store; + const TxnBaseBase *base; + const TxnInsertPage *page; + TxnMemtableVersionVersion *run; + size_t run_len; + StdVec *result; + HostError *error; + bool ok; + StdVec takers; + bool takers_live; + bool has_below; + StdVec below; + StdVec took; + bool took_live; + bool result_live; +} txn_vacuum_leaving_frame_t; + +/* txn_vacuum_leaving_drop frees the takes leaving gives. */ +void +txn_vacuum_leaving_drop(StdVec *leaving) +{ + for (size_t i = 0; i < leaving->len; i++) + txn_vacuum_ids_drop(&STD_VEC_AT(leaving, TxnVacuumLeaving, i).ids); + std_vec_free_Checked(leaving, sizeof(TxnVacuumLeaving), _Alignof(TxnVacuumLeaving)); +} + +/* txn_vacuum_leaving_pad frees what leaving holds live. */ +static void +txn_vacuum_leaving_pad(void *frame_) +{ + txn_vacuum_leaving_frame_t *frame = frame_; + + if (frame->took_live) + txn_vacuum_took_drop(&frame->took); + if (frame->has_below) + std_vec_free_Checked(&frame->below, 1, 1); + if (frame->takers_live) + txn_memtable_map_takers_drop(&frame->takers); + if (frame->result_live) + txn_vacuum_leaving_drop(frame->result); + frame->took_live = frame->has_below = frame->takers_live = frame->result_live = false; +} + +/* txn_vacuum_leaving_body is leaving's body. */ +static void +txn_vacuum_leaving_body(void *frame_) +{ + txn_vacuum_leaving_frame_t *frame = frame_; + HostRow taken = {0}; + uint32 block; + + std_vec_new(frame->result, sizeof(TxnVacuumLeaving), _Alignof(TxnVacuumLeaving)); + frame->result_live = true; + for (size_t k = 0; k < frame->run_len && taken.f0 == 0; k++) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(&frame->run[k]); + + taken = txn_memtable_map_taken(txn_memtable_version_Record_op(&record)); + } + if (taken.f0 == 0) + { + frame->result_live = false; + frame->ok = true; + return; + } + block = (uint32) (taken.f0 >> 16); + std_vec_new(&frame->takers, sizeof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t), + _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); + frame->takers_live = true; + if (!std_vec_try_reserve_exact(&frame->takers, frame->run_len, sizeof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t), + _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t))) + { + *frame->error = host_Error_out_of_memory(); + return; + } + for (size_t k = frame->run_len; k > 0; k--) + { + txn_TupleWriterRowEntryU64TupleVecU32Bool_t taker; + + if (!txn_memtable_map_taker(&frame->run[k - 1], &taker, frame->error)) + return; + std_vec_push_Checked(&frame->takers, &taker, sizeof(taker), _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); + } + if (!txn_vacuum_below(frame->io, frame->store, frame->base, block, &frame->has_below, &frame->below, + frame->error) || + !txn_vacuum_taken(&frame->takers, frame->has_below, &frame->below, block, frame->base, frame->page, + &frame->took, frame->error)) + return; + frame->took_live = true; + if (!std_vec_try_reserve_exact(frame->result, frame->took.len, sizeof(TxnVacuumLeaving), + _Alignof(TxnVacuumLeaving))) + { + *frame->error = host_Error_out_of_memory(); + return; + } + for (size_t i = 0; i < frame->took.len; i++) + { + TxnVacuumTook *take = &STD_VEC_AT(&frame->took, TxnVacuumTook, i); + TxnVacuumLeaving leaving = {take->f0, take->f1, take->f2, take->f3}; + + if (!txn_vacuum_committed_ids(frame->io, take, &leaving.ids, frame->error)) + return; + std_vec_push_Checked(frame->result, &leaving, sizeof(leaving), _Alignof(TxnVacuumLeaving)); + } + frame->result_live = false; + frame->ok = true; +} + +/* + * txn_vacuum_leaving returns the taken offsets of a run of a block's free + * space versions, newest first, that a fold's drop takes out of the delta, + * as VACUUM's find reads them (taken) against the base's entry below the + * delta: each tuple id with its take's writer, the bytes its take lowered, + * its stored ender, and its row's value ids when the writer whose commit + * decides them committed (committed_ids); a run with no take gives none. + */ +bool +txn_vacuum_leaving(HostIoIo *io, TreesStoreNodeStore store, const TxnBaseBase *base, const TxnInsertPage *page, + TxnMemtableVersionVersion *run, size_t run_len, StdVec *result, HostError *error) +{ + txn_vacuum_leaving_frame_t frame = {.io = io, .store = store, .base = base, .page = page, .run = run, + .run_len = run_len, .result = result, .error = error}; + + std_landing(&frame, txn_vacuum_leaving_body, txn_vacuum_leaving_pad); + txn_vacuum_leaving_pad(&frame); + return frame.ok; +} + /* * txn_vacuum_runs splits tuple ids with their bytes, in tuple id order, into * each block's runs of consecutive offsets. @@ -642,7 +856,13 @@ txn_vacuum_cleared(void *captures, bool has_current, const uint8 *current, size_ return TXN_MEMTABLE_MAP_STEP_PUT; } -/* txn_vacuum_taken_down is a value's step, values' take down by its captured count. */ +/* + * txn_vacuum_taken_down is a value's step, values' take down by its captured + * count. Its take falls back to the newest entry its caller read whole, which + * a spill moves unchanged, and a write between is one the values mark the + * live closure rechecks covers, or in redo none, the startup being the map's + * only writer, so it has no read to compare (bugs row 342). + */ static TxnMemtableMapStep txn_vacuum_taken_down(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out) { @@ -667,6 +887,9 @@ typedef struct txn_vacuum_block_frame_t const TxnBaseBase *base; uint32 block; const TxnInsertPage *page; + /* spilled is a Vec of the tuple ids the block's spilled takes took. */ + StdVec spilled; + bool spilled_live; bool has_below; StdVec below; bool takers_live; @@ -721,8 +944,10 @@ txn_vacuum_block_body(void *frame_) if (!txn_vacuum_below(frame->io, frame->store, frame->base, frame->block, &frame->has_below, &frame->below, frame->error) || !txn_vacuum_block_reread(frame) || - !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-vacuum-found", frame->error)) + !host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-vacuum-found", frame->error) || + !txn_memtable_map_spilled_takes(frame->io, frame->record, frame->block, &frame->spilled, frame->error)) return; + frame->spilled_live = true; if (!txn_vacuum_taken(&frame->takers, frame->has_below, &frame->below, frame->block, frame->base, frame->page, &frame->took, frame->error)) return; @@ -738,7 +963,7 @@ txn_vacuum_block_body(void *frame_) if (!host_in_progress(HOST_IO_IO_IO_DOORS(frame->io), top, &running, frame->error)) return; - if (running) + if (running || txn_vacuum_spilled(&frame->spilled, take->f0)) continue; if (!txn_vacuum_mapped(frame->io, frame->store, frame->record, txn_vacuum_tuple_ids(frame->base), take->f0, &mapped, frame->error)) @@ -758,24 +983,11 @@ txn_vacuum_block_body(void *frame_) TxnVacuumTook *take = &STD_VEC_AT(&frame->took, TxnVacuumTook, i); TxnVacuumFound found = {.f0 = take->f0, .f1 = take->f2, .f2 = take->f3, .f3 = take->f3 == take->f1.xid ? 0 : take->f3}; - HostStatus status = HOST_STATUS_ABORTED; if (!txn_vacuum_contains(&frame->free, take->f0)) continue; - if (take->ids.len > 0) - { - HostXid named = {take->named}; - - if (!host_status(HOST_IO_IO_IO_DOORS(frame->io), named, &status, frame->error)) - return; - } - if (status == HOST_STATUS_COMMITTED) - { - found.f4 = take->ids; - std_vec_new(&take->ids, TXN_VACUUM_ID); - } - else - std_vec_new(&found.f4, TXN_VACUUM_ID); + if (!txn_vacuum_committed_ids(frame->io, take, &found.f4, frame->error)) + return; std_vec_push_Checked(frame->result, &found, TXN_VACUUM_FOUND); } } @@ -795,11 +1007,14 @@ txn_vacuum_block_pad(void *frame_) txn_vacuum_took_drop(&frame->took); if (frame->takers_live) txn_memtable_map_takers_drop(&frame->takers); + if (frame->spilled_live) + std_vec_free_Checked(&frame->spilled, sizeof(HostRow), _Alignof(HostRow)); if (frame->has_below) std_vec_free_Checked(&frame->below, 1, 1); if (frame->result_live) txn_vacuum_found_drop(frame->result); - frame->free_live = frame->took_live = frame->takers_live = frame->has_below = frame->result_live = false; + frame->free_live = frame->took_live = frame->takers_live = frame->spilled_live = frame->has_below = + frame->result_live = false; } /* @@ -1039,8 +1254,8 @@ txn_vacuum_pended_body(void *frame_) merged = &STD_VEC_AT(frame->found, TxnVacuumFound, i); if (merged != NULL) { - merged->f2 = Max(merged->f2, pending->ender); - merged->f3 = Max(merged->f3, pending->ender); + merged->f2 = Max(merged->f2, pending->ender & ~TXN_BASE_OWN); + merged->f3 = Max(merged->f3, txn_vacuum_logged(pending->ender)); if (merged->f4.len == 0) { txn_vacuum_ids_drop(&merged->f4); @@ -1059,8 +1274,8 @@ txn_vacuum_pended_body(void *frame_) continue; found.f0 = pending->tid; found.f1 = pending->bytes; - found.f2 = pending->ender; - found.f3 = pending->ender; + found.f2 = pending->ender & ~TXN_BASE_OWN; + found.f3 = txn_vacuum_logged(pending->ender); found.f4 = pending->ids; std_vec_push_Checked(frame->found, &found, TXN_VACUUM_FOUND); std_vec_new(&pending->ids, TXN_VACUUM_ID); @@ -1083,8 +1298,10 @@ txn_vacuum_pended_pad(void *frame_) /* * txn_vacuum_pended adds to VACUUM's found tuple ids those of a base's - * pending tree, each with its heap bytes and its row's ender's full xid, - * that no layer maps and that the free space map doesn't already hold free, + * pending tree, each with its heap bytes, its row's ender's full xid, and its + * free's conflict horizon, that xid, or 0 for a row its own writer ended + * (TXN_BASE_OWN), that no layer maps and that the free space map doesn't + * already hold free, * so a tuple id is freed once; one the delta's takes found already keeps the * larger of the two enders, and its take's value ids, taking the entry's * only when the take holds none. It keeps the found tuple ids in order. @@ -1161,7 +1378,6 @@ txn_vacuum_never_held_body(void *frame_) { txn_TupleU32Entry_t seek; bool some; - uint64 moved; if (!txn_memtable_map_seek(frame->io, record, TXN_MEMTABLE_MAP_MAP_FREE, from, &some, &seek, frame->error)) return; @@ -1173,10 +1389,9 @@ txn_vacuum_never_held_body(void *frame_) seek.f0, frame->page, &frame->block, frame->error)) return; frame->block_live = true; - moved = txn_state_generation(record); - if (moved != generation) + if (txn_state_moved_past(record, generation)) { - generation = moved; + generation = txn_state_generation(record); frame->block_live = false; txn_vacuum_found_drop(&frame->block); if (!txn_vacuum_never_held_base(frame)) @@ -1207,7 +1422,7 @@ txn_vacuum_never_held_body(void *frame_) return; if (removable) { - TxnVacuumFreed passed = {found->f0, found->f1, found->f3, found->f4}; + TxnVacuumFreed passed = {found->f0, found->f1, found->f3, found->f4, txn_vacuum_stored(found->f2, found->f3)}; std_vec_new(&found->f4, TXN_VACUUM_ID); std_vec_push_Checked(frame->result, &passed, sizeof(passed), _Alignof(TxnVacuumFreed)); @@ -1287,7 +1502,26 @@ typedef struct txn_vacuum_spent_frame_t bool ok; } txn_vacuum_spent_frame_t; -/* txn_vacuum_spent_body counts the block's never-held tuple ids, then its pending ones no layer maps or map frees. */ +/* txn_vacuum_spent_pad frees what spent holds live. */ +static void +txn_vacuum_spent_pad(void *frame_) +{ + txn_vacuum_spent_frame_t *frame = frame_; + + if (frame->pending_live) + std_vec_free_Checked(&frame->pending, sizeof(uint64), _Alignof(uint64)); + if (frame->taken_live) + txn_vacuum_found_drop(&frame->taken); + if (frame->base_live) + txn_base_Base_drop(&frame->base); + frame->pending_live = frame->taken_live = frame->base_live = false; +} + +/* + * txn_vacuum_spent_body counts the block's never-held tuple ids, then its + * pending ones no layer maps or map frees, and counts again under the newest + * base when an install moved the base before it finished. + */ static void txn_vacuum_spent_body(void *frame_) { @@ -1295,59 +1529,52 @@ txn_vacuum_spent_body(void *frame_) HostSharedRecord record = host_shared_Held_record(frame->held); bool some; - if (!txn_state_base_record(frame->io, frame->held, &some, &frame->base, frame->error)) - return; - frame->base_live = some; - if (!txn_vacuum_block_never_held(frame->io, frame->store, record, some ? &frame->base : NULL, frame->block, - frame->page, &frame->taken, frame->error)) - return; - frame->taken_live = true; - *frame->result = (uint64) frame->taken.len; - if (some && frame->base.has_pending) + for (;;) { - std_vec_new(&frame->pending, sizeof(uint64), _Alignof(uint64)); - frame->pending_live = true; - if (!txn_vacuum_pending_in(frame->io, frame->store, frame->base.pending, frame->block, &frame->pending, - frame->error)) + uint64 generation = txn_state_generation(record); + + if (!txn_state_base_record(frame->io, frame->held, &some, &frame->base, frame->error)) + return; + frame->base_live = some; + if (!txn_vacuum_block_never_held(frame->io, frame->store, record, some ? &frame->base : NULL, frame->block, + frame->page, &frame->taken, frame->error)) return; - for (size_t i = 0; i < frame->pending.len; i++) + frame->taken_live = true; + *frame->result = (uint64) frame->taken.len; + if (some && frame->base.has_pending) { - uint64 tid = STD_VEC_AT(&frame->pending, uint64, i); - bool taken = false; - bool skip; - - for (size_t at = 0; at < frame->taken.len && !taken; at++) - taken = STD_VEC_AT(&frame->taken, TxnVacuumFound, at).f0 == tid; - if (taken) - continue; - if (!txn_vacuum_mapped(frame->io, frame->store, record, txn_vacuum_tuple_ids(&frame->base), tid, &skip, - frame->error)) - return; - if (!skip && !txn_vacuum_frees(frame->io, frame->store, frame->held, &frame->base, tid, &skip, - frame->error)) + std_vec_new(&frame->pending, sizeof(uint64), _Alignof(uint64)); + frame->pending_live = true; + if (!txn_vacuum_pending_in(frame->io, frame->store, frame->base.pending, frame->block, &frame->pending, + frame->error)) return; - if (!skip) - *frame->result = STD_ADD(u64, *frame->result, 1, 189, 17); + for (size_t i = 0; i < frame->pending.len; i++) + { + uint64 tid = STD_VEC_AT(&frame->pending, uint64, i); + bool taken = false; + bool skip; + + for (size_t at = 0; at < frame->taken.len && !taken; at++) + taken = STD_VEC_AT(&frame->taken, TxnVacuumFound, at).f0 == tid; + if (taken) + continue; + if (!txn_vacuum_mapped(frame->io, frame->store, record, txn_vacuum_tuple_ids(&frame->base), tid, &skip, + frame->error)) + return; + if (!skip && !txn_vacuum_frees(frame->io, frame->store, frame->held, &frame->base, tid, &skip, + frame->error)) + return; + if (!skip) + *frame->result = STD_ADD(u64, *frame->result, 1, 189, 17); + } } + if (!txn_state_moved_past(record, generation)) + break; + txn_vacuum_spent_pad(frame); } frame->ok = true; } -/* txn_vacuum_spent_pad frees what spent holds live. */ -static void -txn_vacuum_spent_pad(void *frame_) -{ - txn_vacuum_spent_frame_t *frame = frame_; - - if (frame->pending_live) - std_vec_free_Checked(&frame->pending, sizeof(uint64), _Alignof(uint64)); - if (frame->taken_live) - txn_vacuum_found_drop(&frame->taken); - if (frame->base_live) - txn_base_Base_drop(&frame->base); - frame->pending_live = frame->taken_live = frame->base_live = false; -} - /* * txn_vacuum_spent counts a block's spent places, which ANALYZE counts dead * as heap counts an aborted insert's tuples and an ended row's until VACUUM @@ -1418,6 +1645,96 @@ txn_vacuum_marker_tuple(void *captures) return tuple; } +/* + * txn_vacuum_Read_t is what a VACUUM free space take reads of its block + * outside its window, which its finish compares and a reread renews (bugs + * row 342): the base, the entry below the memtable, the claims' start, the + * memtable's newest entry, else the one below, and the base generation they + * were read at. + */ +typedef struct txn_vacuum_Read_t +{ + HostIoIo *io; + TreesStoreNodeStore store; + const HostSharedHeld *held; + uint32 block; + const TxnInsertPage *page; + uint64 generation; + bool base_live; + TxnBaseBase base; + bool has_below; + StdVec below; + uint64 start; + bool has_current; + StdVec current; +} txn_vacuum_Read_t; + +/* txn_vacuum_Read_drop frees what a read holds. */ +static void +txn_vacuum_Read_drop(txn_vacuum_Read_t *self) +{ + if (self->base_live) + txn_base_Base_drop(&self->base); + if (self->has_below) + std_vec_free_Checked(&self->below, 1, 1); + if (self->has_current) + std_vec_free_Checked(&self->current, 1, 1); + self->base_live = self->has_below = self->has_current = false; +} + +/* + * txn_vacuum_read reads a block's entries for a VACUUM take after the base + * generation: the base record, its entry and the claims' start from it, the + * spills' newest in its place, and the memtable's newest, pausing on a first + * read at doltgres-vacuum-freeing after the base record and at + * doltgres-vacuum-read after the rest. + */ +static bool +txn_vacuum_read(txn_vacuum_Read_t *self, bool first, HostError *error) +{ + HostSharedRecord record = host_shared_Held_record(self->held); + TxnMemtableMapEntry newest; + bool some; + + txn_vacuum_Read_drop(self); + self->generation = txn_state_generation(record); + if (!txn_state_base_record(self->io, self->held, &some, &self->base, error)) + return false; + self->base_live = some; + if ((first && !host_pause(HOST_SHARED_IO_PAUSES(self->io), "doltgres-vacuum-freeing", error)) || + !txn_vacuum_below(self->io, self->store, self->base_live ? &self->base : NULL, self->block, + &self->has_below, &self->below, error) || + !txn_vacuum_first(self->has_below, &self->below, self->block, self->base_live ? &self->base : NULL, + self->page, &self->start, error) || + !txn_vacuum_spilled_below(self->io, record, self->block, &self->has_below, &self->below, error) || + !txn_memtable_map_in_memtable(self->io, record, TXN_MEMTABLE_MAP_MAP_FREE, self->block, &some, &newest, + error)) + return false; + if (some) + { + self->has_current = newest.has_value; + if (newest.has_value) + self->current = newest.value; + } + else if (self->has_below) + { + std_slice_to_vec_Checked(&self->current, self->below.ptr, self->below.len, 1, 1); + self->has_current = true; + } + return !first || host_pause(HOST_SHARED_IO_PAUSES(self->io), "doltgres-vacuum-read", error); +} + +/* txn_vacuum_Read_map is a read's entry and generation, as a take compares them. */ +static TxnMemtableMapRead +txn_vacuum_Read_map(const txn_vacuum_Read_t *self) +{ + return (TxnMemtableMapRead) + { + .has_entry = self->has_current, .entry = self->has_current ? self->current.ptr : NULL, + .entry_len = self->has_current ? self->current.len : 0, .generation = self->generation + }; +} + /* * txn_vacuum_given_closure_2_captures_t is given's prepare's closure's * captures: the block's take's parts, its runs and markers, the values it @@ -1427,9 +1744,7 @@ typedef struct txn_vacuum_given_closure_2_captures_t { HostSharedRecord record; uint32 block; - bool has_below; - const StdVec *below; - uint64 start; + txn_vacuum_Read_t *read; const TxnInsertPage *page; const StdVec *freed; const StdVec *markers; @@ -1467,8 +1782,9 @@ typedef struct txn_vacuum_given_closure_2_frame_t } txn_vacuum_given_closure_2_frame_t; /* - * txn_vacuum_given_closure_2_body checks the values mark, logs the record, and - * makes the block's and the values' takes. + * txn_vacuum_given_closure_2_body checks the base generation and the values + * mark, holds the block's range and checks its entry there before a logged + * window, logs the record, and makes the block's and the values' takes. */ static void txn_vacuum_given_closure_2_body(void *frame_) @@ -1482,17 +1798,38 @@ txn_vacuum_given_closure_2_body(void *frame_) HostLsn lsn = c->lsn; size_t freeing = c->freed->len > 0; TxnMemtableMapTaken taken; + TxnStatusRoomed unroomed = {0}; + txn_vacuum_Read_t *read = c->read; + TxnMemtableMapRead map; + const TxnMemtableMapRead *reading = ↦ + bool moved = txn_state_generation(c->record) != read->generation; - if (c->toast != NULL) + if (!moved && c->toast != NULL) { std_TupleU64U64_t mark = txn_values_mark(*c->toast); - if (mark.f0 != c->mark.f0 || mark.f1 != c->mark.f1) - { - *frame->result = false; - frame->ok = true; + moved = mark.f0 != c->mark.f0 || mark.f1 != c->mark.f1; + } + if (!moved && c->windowed && freeing + c->markers->len > 0) + { + bool unmoved; + + /* + * The block's takes run on its kept range with no read, so the window + * reads and waits on nothing (bugs row 342). + */ + map = txn_vacuum_Read_map(read); + if (!txn_memtable_map_kept(io, frame->prepared, 0, c->record, read->has_below, read->below.ptr, + read->below.len, &map, &unmoved, frame->error)) return; - } + moved = !unmoved; + reading = NULL; + } + if (moved) + { + *frame->result = false; + frame->ok = true; + return; } if (c->windowed) { @@ -1515,10 +1852,16 @@ txn_vacuum_given_closure_2_body(void *frame_) txn_vacuum_BlockStep_t step = {.record = c->record, .block = c->block, .runs = c->freed->ptr, .runs_len = c->freed->len, .most = c->page->most}; - if (!txn_memtable_map_finish_Unit(io, frame->prepared, 0, wrote, has_lsn, lsn, c->has_below, c->below->ptr, - c->below->len, txn_vacuum_given_back, &step, NULL, txn_vacuum_nothing, NULL, - NULL, &taken, frame->error)) - return; + do + { + map = txn_vacuum_Read_map(read); + if (!txn_memtable_map_finish_Unit(io, frame->prepared, 0, wrote, false, unroomed, has_lsn, lsn, + read->has_below, read->below.ptr, read->below.len, reading, + txn_vacuum_given_back, &step, NULL, txn_vacuum_nothing, NULL, NULL, + &taken, frame->error) || + (taken == TXN_MEMTABLE_MAP_TAKEN_MOVED && !txn_vacuum_read(read, false, frame->error))) + return; + } while (taken == TXN_MEMTABLE_MAP_TAKEN_MOVED); if (taken != TXN_MEMTABLE_MAP_TAKEN_TOOK) { *frame->error = txn_vacuum_corrupt(); @@ -1532,12 +1875,17 @@ txn_vacuum_given_closure_2_body(void *frame_) .most = c->page->most}; HostRow tid = {marker->f0}; - if (!txn_memtable_map_took_Unit(io, frame->prepared, freeing + at, wrote, has_lsn, lsn, true, tid, c->start, - c->has_below, c->below->ptr, c->below->len, txn_vacuum_marked, &step, NULL, - txn_vacuum_marker_tuple, (void *) &marker->f0, NULL, txn_vacuum_nothing, NULL, - NULL, &taken, - frame->error)) - return; + do + { + map = txn_vacuum_Read_map(read); + if (!txn_memtable_map_took_Unit(io, frame->prepared, freeing + at, wrote, false, unroomed, has_lsn, lsn, + true, tid, read->start, read->has_below, read->below.ptr, read->below.len, + reading, txn_vacuum_marked, &step, NULL, txn_vacuum_marker_tuple, + (void *) &marker->f0, NULL, txn_vacuum_nothing, NULL, NULL, &taken, + frame->error) || + (taken == TXN_MEMTABLE_MAP_TAKEN_MOVED && !txn_vacuum_read(read, false, frame->error))) + return; + } while (taken == TXN_MEMTABLE_MAP_TAKEN_MOVED); if (taken == TXN_MEMTABLE_MAP_TAKEN_TOOK) STD_VEC_AT(c->claims, txn_vacuum_TupleU64Bool_t, at).f1 = true; else if (taken == TXN_MEMTABLE_MAP_TAKEN_RETRY) @@ -1546,12 +1894,14 @@ txn_vacuum_given_closure_2_body(void *frame_) return; } } + txn_memtable_table_unkept(frame->prepared); for (size_t at = 0; c->toast != NULL && at < c->downs->len; at++) { TxnVacuumDown *down = &STD_VEC_AT(c->downs, TxnVacuumDown, at); - if (!txn_memtable_map_finish_Unit(io, frame->prepared, freeing + c->markers->len + at, wrote, has_lsn, lsn, - down->has_based, down->based.ptr, down->based.len, txn_vacuum_taken_down, + if (!txn_memtable_map_finish_Unit(io, frame->prepared, freeing + c->markers->len + at, wrote, false, unroomed, + has_lsn, lsn, + down->has_based, down->based.ptr, down->based.len, NULL, txn_vacuum_taken_down, &down->by, NULL, txn_vacuum_nothing, NULL, NULL, &taken, frame->error)) return; if (taken != TXN_MEMTABLE_MAP_TAKEN_TOOK) @@ -1569,7 +1919,7 @@ txn_vacuum_given_closure_2_body(void *frame_) frame->ok = true; } -/* txn_vacuum_given_closure_2_pad drops the closure's window, while live. */ +/* txn_vacuum_given_closure_2_pad drops the closure's window and its block's kept range, while live. */ static void txn_vacuum_given_closure_2_pad(void *frame_) { @@ -1578,6 +1928,7 @@ txn_vacuum_given_closure_2_pad(void *frame_) if (frame->window_live) host_DelayCheckpoint_drop(&frame->window); frame->window_live = false; + txn_memtable_table_unkept(frame->prepared); } /* txn_vacuum_given_closure_2 is given's prepare's closure. */ @@ -1688,8 +2039,9 @@ txn_vacuum_given_pad(void *frame_) * version is already a marker; it answers which dead tuple ids it claimed, * in order. It takes down, in the same prepare and window, each value id in * its TOAST storage's value map by its count, and answers done false, before - * it opens the window, when the TOAST storage's mark moved, for its caller to - * read them again. + * it opens the window, when an install moved the base since the block's entry + * was read, a spill or another take moved the entry, or the TOAST storage's + * mark moved, for its caller to read them again. */ static bool txn_vacuum_given(HostIoIo *io, const txn_vacuum_given_closure_2_captures_t *captures, const StdVec *dead, @@ -1743,8 +2095,6 @@ typedef struct txn_vacuum_freed_frame_t const TxnVacuumFreed *tids; size_t tids_len; bool dead; - bool base_live; - TxnBaseBase base; bool kept_live; StdVec kept; bool runs_live; @@ -1753,14 +2103,20 @@ typedef struct txn_vacuum_freed_frame_t StdVec ids; bool counts_live; StdVec counts; - bool has_below; - StdVec below; + txn_vacuum_Read_t read; bool downs_live; StdVec downs; bool body_live; StdVec body; bool claims_live; StdVec claims; + + /* + * enders is a Vec: the stored enders of a block's freed tuple ids, + * which its record names. + */ + bool enders_live; + StdVec enders; StdVec *result; bool result_live; HostError *error; @@ -1777,14 +2133,15 @@ txn_vacuum_freed_block_drop(txn_vacuum_freed_frame_t *frame) std_vec_free_Checked(&frame->body, 1, 1); if (frame->downs_live) txn_vacuum_downs_drop(&frame->downs); - if (frame->has_below) - std_vec_free_Checked(&frame->below, 1, 1); + txn_vacuum_Read_drop(&frame->read); if (frame->counts_live) std_vec_free_Checked(&frame->counts, TXN_VACUUM_COUNT); if (frame->ids_live) std_vec_free_Checked(&frame->ids, TXN_VACUUM_ID); - frame->claims_live = frame->body_live = frame->downs_live = frame->has_below = false; - frame->counts_live = frame->ids_live = false; + if (frame->enders_live) + std_vec_free_Checked(&frame->enders, sizeof(uint64), _Alignof(uint64)); + frame->claims_live = frame->body_live = frame->downs_live = false; + frame->counts_live = frame->ids_live = frame->enders_live = false; } /* txn_vacuum_freed_counts counts how many of a block's freed rows named each value id, ascending by id. */ @@ -1823,7 +2180,11 @@ txn_vacuum_freed_counts(txn_vacuum_freed_frame_t *frame, uint32 block) } } -/* txn_vacuum_freed_block gives back one block's runs, reading its values' entries again until their mark holds. */ +/* + * txn_vacuum_freed_block gives back one block's runs, reading the base and the + * block's and its values' entries again until neither the base nor their mark + * moved; it pauses at doltgres-vacuum-freeing after each read of the base. + */ static bool txn_vacuum_freed_block(txn_vacuum_freed_frame_t *frame, const std_TupleU32U16U16U64_t *runs, size_t runs_len) { @@ -1843,6 +2204,15 @@ txn_vacuum_freed_block(txn_vacuum_freed_frame_t *frame, const std_TupleU32U16U16 horizon = Max(horizon, freed->f2); } txn_vacuum_freed_counts(frame, block); + std_vec_new(&frame->enders, sizeof(uint64), _Alignof(uint64)); + frame->enders_live = true; + for (size_t i = 0; i < frame->kept.len; i++) + { + const TxnVacuumFreed *freed = STD_VEC_AT(&frame->kept, const TxnVacuumFreed *, i); + + if ((uint32) (freed->f0 >> 16) == block) + std_vec_push_Checked(&frame->enders, &freed->f4, sizeof(uint64), _Alignof(uint64)); + } toasted = frame->counts.len > 0; if (toasted && frame->toast == NULL) { @@ -1853,18 +2223,19 @@ txn_vacuum_freed_block(txn_vacuum_freed_frame_t *frame, const std_TupleU32U16U16 toast_record = host_shared_Held_record(frame->toast); for (;;) { - txn_vacuum_given_closure_2_captures_t captures = {.record = record, .block = block, .below = &frame->below, + txn_vacuum_given_closure_2_captures_t captures = {.record = record, .block = block, .read = &frame->read, .page = frame->page, .freed = frame->dead ? &none : &these, .toast = toasted ? &toast_record : NULL, .downs = &frame->downs, .windowed = true}; bool has_mark = false; bool done; - if (!txn_vacuum_below(frame->io, frame->store, frame->base_live ? &frame->base : NULL, block, - &frame->has_below, &frame->below, frame->error) || - !txn_vacuum_first(frame->has_below, &frame->below, block, frame->base_live ? &frame->base : NULL, - frame->page, &captures.start, frame->error)) + frame->read.io = frame->io; + frame->read.store = frame->store; + frame->read.held = frame->held; + frame->read.block = block; + frame->read.page = frame->page; + if (!txn_vacuum_read(&frame->read, true, frame->error)) return false; - captures.has_below = frame->has_below; std_vec_with_capacity_Checked(&frame->downs, frame->counts.len, TXN_VACUUM_DOWN); frame->downs_live = true; for (size_t i = 0; toasted && i < frame->counts.len; i++) @@ -1886,7 +2257,8 @@ txn_vacuum_freed_block(txn_vacuum_freed_frame_t *frame, const std_TupleU32U16U16 if (frame->logged) { TxnRecordTidsFreed record_ = {.storage = frame->storage, .horizon = horizon, - .runs = frame->dead ? none : these, .dead = frame->dead ? these : none, .has_values = toasted, + .runs = frame->dead ? none : these, .dead = frame->dead ? these : none, .enders = frame->enders, + .has_values = toasted, .values = {.f0 = frame->toast_storage, .f1 = frame->counts}}; txn_record_TidsFreed_encode(&record_, &frame->body); @@ -1916,9 +2288,6 @@ txn_vacuum_freed_block(txn_vacuum_freed_frame_t *frame, const std_TupleU32U16U16 frame->body_live = false; frame->downs_live = false; txn_vacuum_downs_drop(&frame->downs); - if (frame->has_below) - std_vec_free_Checked(&frame->below, 1, 1); - frame->has_below = false; } } @@ -1927,13 +2296,9 @@ static void txn_vacuum_freed_body(void *frame_) { txn_vacuum_freed_frame_t *frame = frame_; - bool some; std_vec_new(frame->result, sizeof(uint64), _Alignof(uint64)); frame->result_live = true; - if (!txn_state_base_record(frame->io, frame->held, &some, &frame->base, frame->error)) - return; - frame->base_live = some; std_vec_new(&frame->kept, sizeof(const TxnVacuumFreed *), _Alignof(const TxnVacuumFreed *)); frame->kept_live = true; for (size_t i = 0; i < frame->tids_len; i++) @@ -1971,11 +2336,9 @@ txn_vacuum_freed_pad(void *frame_) std_vec_free_Checked(&frame->runs, TXN_VACUUM_RUN); if (frame->kept_live) std_vec_free_Checked(&frame->kept, sizeof(const TxnVacuumFreed *), _Alignof(const TxnVacuumFreed *)); - if (frame->base_live) - txn_base_Base_drop(&frame->base); if (frame->result_live) std_vec_free_Checked(frame->result, sizeof(uint64), _Alignof(uint64)); - frame->runs_live = frame->kept_live = frame->base_live = frame->result_live = false; + frame->runs_live = frame->kept_live = frame->result_live = false; } /* @@ -2028,25 +2391,57 @@ txn_vacuum_removals_drop(StdVec *removals) std_vec_free_Checked(removals, TXN_VACUUM_REMOVAL); } -/* txn_vacuum_pruned_closure_8_captures_t is a prune's clear's prepare's closure's captures. */ +/* + * txn_vacuum_pruned_closure_8_captures_t is a prune's clear's prepare's + * closure's captures, with the base generation its entry below was read at, + * and moved, which the closure sets when an install moved the base, or the + * entry moved, since. + */ typedef struct txn_vacuum_pruned_closure_8_captures_t { txn_vacuum_BlockStep_t step; + uint64 generation; + bool has_entry; + + /* + * entry is the block's newest entry the prune read, which the take + * compares (bugs row 342). + */ + const StdVec *entry; bool has_below; const StdVec *below; + bool moved; } txn_vacuum_pruned_closure_8_captures_t; -/* txn_vacuum_pruned_closure_8 is a prune's clear's prepare's closure: the block's take that clears its full flag. */ +/* + * txn_vacuum_pruned_closure_8 is a prune's clear's prepare's closure: the + * block's take that clears its full flag, or none when an install moved the + * base, or a spill or another take moved the entry, since its entry was read. + */ static bool txn_vacuum_pruned_closure_8(void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, TxnMemtableMapTaken *result, HostError *error) { txn_vacuum_pruned_closure_8_captures_t *inside = captures; TxnMemtableVersionWriter none = TXN_MEMTABLE_VERSION_NO_TRANSACTION; + TxnStatusRoomed unroomed = {0}; + + TxnMemtableMapRead read = {.has_entry = inside->has_entry, .entry = inside->has_entry ? inside->entry->ptr : NULL, + .entry_len = inside->has_entry ? inside->entry->len : 0, .generation = inside->generation}; - return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, none, false, 0, inside->has_below, - inside->below->ptr, inside->below->len, txn_vacuum_cleared, &inside->step, NULL, - txn_vacuum_nothing, NULL, NULL, result, error); + inside->moved = txn_state_generation(inside->step.record) != inside->generation; + if (inside->moved) + { + *result = TXN_MEMTABLE_MAP_TAKEN_RETRY; + return true; + } + if (!txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, none, false, unroomed, false, 0, + inside->has_below, inside->below->ptr, inside->below->len, &read, + txn_vacuum_cleared, &inside->step, NULL, txn_vacuum_nothing, NULL, NULL, result, + error)) + return false; + inside->moved = *result == TXN_MEMTABLE_MAP_TAKEN_MOVED; + return true; } /* @@ -2060,6 +2455,9 @@ typedef struct txn_vacuum_pruned_frame_t HostIoIo *io; TreesStoreNodeStore store; const HostSharedHeld *held; + /* spilled is a Vec of the tuple ids the block's spilled takes took. */ + StdVec spilled; + bool spilled_live; HostStorage storage; bool logged; const TxnInsertPage *page; @@ -2179,9 +2577,43 @@ txn_vacuum_pruned_ended(txn_vacuum_pruned_frame_t *frame, uint64 tid, bool *ende return true; } +/* txn_vacuum_pruned_pad frees what pruned holds live. */ +static void +txn_vacuum_pruned_pad(void *frame_) +{ + txn_vacuum_pruned_frame_t *frame = frame_; + + txn_vacuum_pruned_key_drop(frame); + if (frame->spilled_live) + std_vec_free_Checked(&frame->spilled, sizeof(HostRow), _Alignof(HostRow)); + frame->spilled_live = false; + if (frame->claimed_live) + std_vec_free_Checked(&frame->claimed, sizeof(uint64), _Alignof(uint64)); + if (frame->removals_live) + txn_vacuum_removals_drop(&frame->removals); + if (frame->dead_live) + txn_vacuum_freed_drop(&frame->dead); + if (frame->took_live) + txn_vacuum_took_drop(&frame->took); + if (frame->takers_live) + txn_memtable_map_takers_drop(&frame->takers); + if (frame->newest_live && frame->newest.has_value) + std_vec_free_Checked(&frame->newest.value, 1, 1); + if (frame->has_below) + std_vec_free_Checked(&frame->below, 1, 1); + if (frame->base_live) + txn_base_Base_drop(&frame->base); + frame->claimed_live = frame->removals_live = frame->dead_live = frame->took_live = frame->takers_live = false; + frame->newest_live = frame->has_below = frame->base_live = false; +} + /* * txn_vacuum_pruned_body checks the block's prune condition, finds its dead - * tuple ids, marks them, and takes out those it claimed. + * tuple ids, marks them, and takes out those it claimed; only a changer's end + * makes a tuple id dead, so a dead one a prune claims has an ender; a clear of the full + * flag whose base an install moved since its entry was read checks the block + * again from the newest base (bugs row 240). It pauses at + * doltgres-prune-based after each read of the base. */ static void txn_vacuum_pruned_body(void *frame_) @@ -2194,161 +2626,154 @@ txn_vacuum_pruned_body(void *frame_) const StdVec *value; uint64 floor; - if (!txn_state_base_record(frame->io, frame->held, &some, &frame->base, frame->error)) - return; - frame->base_live = some; - if (!txn_vacuum_below(frame->io, frame->store, some ? &frame->base : NULL, frame->block, &frame->has_below, - &frame->below, frame->error) || - !txn_memtable_map_newest(frame->io, record, TXN_MEMTABLE_MAP_MAP_FREE, frame->block, &some, &frame->newest, - frame->error)) - return; - frame->newest_live = some; - has_value = some ? frame->newest.has_value : frame->has_below; - value = some ? &frame->newest.value : &frame->below; - if (!has_value) - { - frame->ok = true; - return; - } - if (!txn_free_space_Free_decode(value->ptr, value->len, frame->block, (uint64) frame->block << 16, - frame->page->most, &entry, frame->error)) - return; - if (!entry.full) + for (;;) { - floor = Max(frame->reserve, STD_ADD(u64, frame->page->bytes, TXN_VACUUM_PAGE_HEADER, 630, 80) / 10); - if ((entry.bytes > TXN_VACUUM_LINE_POINTER ? entry.bytes - TXN_VACUUM_LINE_POINTER : 0) >= floor) + uint64 generation = txn_state_generation(record); + + if (!txn_state_base_record(frame->io, frame->held, &some, &frame->base, frame->error)) + return; + frame->base_live = some; + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-prune-based", frame->error) || + !txn_vacuum_below(frame->io, frame->store, some ? &frame->base : NULL, frame->block, &frame->has_below, + &frame->below, frame->error) || + !txn_memtable_map_newest(frame->io, record, TXN_MEMTABLE_MAP_MAP_FREE, frame->block, &some, &frame->newest, + frame->error)) + return; + frame->newest_live = some; + has_value = some ? frame->newest.has_value : frame->has_below; + value = some ? &frame->newest.value : &frame->below; + if (!has_value) { frame->ok = true; return; } - } - if (!txn_memtable_map_takers(frame->io, record, TXN_MEMTABLE_MAP_MAP_FREE, frame->block, &frame->takers, - frame->error)) - return; - frame->takers_live = true; - std_vec_new(&frame->dead, sizeof(TxnVacuumFreed), _Alignof(TxnVacuumFreed)); - frame->dead_live = true; - std_vec_new(&frame->removals, TXN_VACUUM_REMOVAL); - frame->removals_live = true; - if (!txn_vacuum_taken(&frame->takers, frame->has_below, &frame->below, frame->block, - frame->base_live ? &frame->base : NULL, frame->page, &frame->took, frame->error)) - return; - frame->took_live = true; - for (size_t i = 0; i < frame->took.len; i++) - { - const TxnVacuumTook *take = &STD_VEC_AT(&frame->took, TxnVacuumTook, i); - HostRow tid = {take->f0}; - bool paired = false; - bool ended; - TxnMemtableVersionHandle handle; - uint64 ender; - - if (frame->base_live && frame->base.has_tuple_ids) + if (!txn_free_space_Free_decode(value->ptr, value->len, frame->block, (uint64) frame->block << 16, + frame->page->most, &entry, frame->error)) + return; + if (!entry.full) { - TreesStoreTree tree = frame->base.tuple_ids; - ChunksErrorError failed; - uint8 key[6]; - - txn_base_tuple_id_key(take->f0, key); - if (!trees_get(frame->io, frame->store, TREES_STORE_TREE_NAMED(&tree), key, sizeof(key), &paired, - &frame->value, &failed)) + floor = Max(frame->reserve, STD_ADD(u64, frame->page->bytes, TXN_VACUUM_PAGE_HEADER, 630, 80) / 10); + if ((entry.bytes > TXN_VACUUM_LINE_POINTER ? entry.bytes - TXN_VACUUM_LINE_POINTER : 0) >= floor) { - *frame->error = txn_base_storage(&failed); + frame->ok = true; return; } - frame->value_live = paired; } - if (take->f2 == 0 || paired) - { - txn_vacuum_pruned_key_drop(frame); - continue; - } - if (!txn_memtable_tuples_get(frame->io, record, tid, &some, &frame->key, frame->error)) + if (!txn_memtable_map_takers(frame->io, record, TXN_MEMTABLE_MAP_MAP_FREE, frame->block, &frame->takers, + frame->error)) return; - frame->key_live = some; - if (!some) - continue; - if (!txn_vacuum_pruned_ended(frame, take->f0, &ended, &handle, &ender)) + frame->takers_live = true; + if (!txn_memtable_map_spilled_takes(frame->io, record, frame->block, &frame->spilled, frame->error)) + return; + frame->spilled_live = true; + std_vec_new(&frame->dead, sizeof(TxnVacuumFreed), _Alignof(TxnVacuumFreed)); + frame->dead_live = true; + std_vec_new(&frame->removals, TXN_VACUUM_REMOVAL); + frame->removals_live = true; + if (!txn_vacuum_taken(&frame->takers, frame->has_below, &frame->below, frame->block, + frame->base_live ? &frame->base : NULL, frame->page, &frame->took, frame->error)) return; - if (ended) + frame->took_live = true; + for (size_t i = 0; i < frame->took.len; i++) { - TxnVacuumFreed dead = {take->f0, take->f2, ender}; - TxnVacuumRemoval removal = {take->f0, frame->key, handle}; + const TxnVacuumTook *take = &STD_VEC_AT(&frame->took, TxnVacuumTook, i); + HostRow tid = {take->f0}; + bool paired = false; + bool ended; + TxnMemtableVersionHandle handle; + uint64 ender; + + if (frame->base_live && frame->base.has_tuple_ids) + { + TreesStoreTree tree = frame->base.tuple_ids; + ChunksErrorError failed; + uint8 key[6]; + + txn_base_tuple_id_key(take->f0, key); + if (!trees_get(frame->io, frame->store, TREES_STORE_TREE_NAMED(&tree), key, sizeof(key), &paired, + &frame->value, &failed)) + { + *frame->error = txn_base_storage(&failed); + return; + } + frame->value_live = paired; + } + if (take->f2 == 0 || paired || txn_vacuum_spilled(&frame->spilled, take->f0)) + { + txn_vacuum_pruned_key_drop(frame); + continue; + } + if (!txn_memtable_tuples_get(frame->io, record, tid, &some, &frame->key, frame->error)) + return; + frame->key_live = some; + if (!some) + continue; + if (!txn_vacuum_pruned_ended(frame, take->f0, &ended, &handle, &ender)) + return; + if (ended) + { + TxnVacuumFreed dead = {take->f0, take->f2, ender, .f4 = ender == take->f1.xid ? ender | TXN_BASE_OWN : ender}; + TxnVacuumRemoval removal = {take->f0, frame->key, handle}; - std_vec_new(&dead.f3, TXN_VACUUM_ID); - std_vec_push_Checked(&frame->dead, &dead, sizeof(dead), _Alignof(TxnVacuumFreed)); - std_vec_push_Checked(&frame->removals, &removal, TXN_VACUUM_REMOVAL); - frame->key_live = false; + std_vec_new(&dead.f3, TXN_VACUUM_ID); + std_vec_push_Checked(&frame->dead, &dead, sizeof(dead), _Alignof(TxnVacuumFreed)); + std_vec_push_Checked(&frame->removals, &removal, TXN_VACUUM_REMOVAL); + frame->key_live = false; + } + txn_vacuum_pruned_key_drop(frame); } - txn_vacuum_pruned_key_drop(frame); - } - if (frame->dead.len > 0) - { - HostStorage none = {0}; + if (frame->dead.len > 0) + { + HostStorage none = {0}; - if (!txn_vacuum_freed(frame->io, frame->store, frame->held, NULL, none, frame->storage, frame->logged, - frame->page, frame->dead.ptr, frame->dead.len, true, &frame->claimed, frame->error)) - return; - frame->claimed_live = true; - if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-prune-marked", frame->error)) + if (!txn_vacuum_freed(frame->io, frame->store, frame->held, NULL, none, frame->storage, frame->logged, + frame->page, frame->dead.ptr, frame->dead.len, true, &frame->claimed, frame->error)) + return; + frame->claimed_live = true; + if (!host_pause(HOST_SHARED_IO_PAUSES(frame->io), "doltgres-prune-marked", frame->error)) + return; + for (size_t i = 0; i < frame->removals.len; i++) + { + TxnVacuumRemoval *removal = &STD_VEC_AT(&frame->removals, TxnVacuumRemoval, i); + bool claimed = false; + + for (size_t at = 0; at < frame->claimed.len && !claimed; at++) + claimed = STD_VEC_AT(&frame->claimed, uint64, at) == removal->tid; + if (!claimed) + continue; + if (!txn_memtable_table_removed(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, frame->io, record, removal->key.ptr, + removal->key.len, &removal->handle, 1, &frame->removed, frame->error)) + return; + std_vec_free_Checked(&frame->removed, sizeof(bool), _Alignof(bool)); + } + frame->ok = true; return; - for (size_t i = 0; i < frame->removals.len; i++) + } + if (entry.full) { - TxnVacuumRemoval *removal = &STD_VEC_AT(&frame->removals, TxnVacuumRemoval, i); - bool claimed = false; + txn_vacuum_pruned_closure_8_captures_t inside = {.step = {.record = record, .block = frame->block, + .most = frame->page->most}, .generation = generation, .has_entry = has_value, .entry = value, + .has_below = frame->has_below, .below = &frame->below}; + TxnMemtableMapTake take = {.state = record, .map = TXN_MEMTABLE_MAP_MAP_FREE, .key = frame->block, + .most = txn_free_space_most_len(frame->page->most)}; + TxnMemtableMapTaken taken; - for (size_t at = 0; at < frame->claimed.len && !claimed; at++) - claimed = STD_VEC_AT(&frame->claimed, uint64, at) == removal->tid; - if (!claimed) - continue; - if (!txn_memtable_table_removed(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, frame->io, record, removal->key.ptr, - removal->key.len, &removal->handle, 1, &frame->removed, frame->error)) + if (!txn_vacuum_spilled_below(frame->io, record, frame->block, &frame->has_below, &frame->below, + frame->error)) return; - std_vec_free_Checked(&frame->removed, sizeof(bool), _Alignof(bool)); + inside.has_below = frame->has_below; + if (!txn_memtable_map_prepare_Taken(frame->io, &take, 1, txn_vacuum_pruned_closure_8, &inside, NULL, + &taken, frame->error)) + return; + if (inside.moved) + { + txn_vacuum_pruned_pad(frame); + continue; + } } frame->ok = true; return; } - if (entry.full) - { - txn_vacuum_pruned_closure_8_captures_t inside = {.step = {.record = record, .block = frame->block, - .most = frame->page->most}, .has_below = frame->has_below, .below = &frame->below}; - TxnMemtableMapTake take = {.state = record, .map = TXN_MEMTABLE_MAP_MAP_FREE, .key = frame->block, - .most = txn_free_space_most_len(frame->page->most)}; - TxnMemtableMapTaken taken; - - if (!txn_memtable_map_prepare_Taken(frame->io, &take, 1, txn_vacuum_pruned_closure_8, &inside, NULL, &taken, - frame->error)) - return; - } - frame->ok = true; -} - -/* txn_vacuum_pruned_pad frees what pruned holds live. */ -static void -txn_vacuum_pruned_pad(void *frame_) -{ - txn_vacuum_pruned_frame_t *frame = frame_; - - txn_vacuum_pruned_key_drop(frame); - if (frame->claimed_live) - std_vec_free_Checked(&frame->claimed, sizeof(uint64), _Alignof(uint64)); - if (frame->removals_live) - txn_vacuum_removals_drop(&frame->removals); - if (frame->dead_live) - txn_vacuum_freed_drop(&frame->dead); - if (frame->took_live) - txn_vacuum_took_drop(&frame->took); - if (frame->takers_live) - txn_memtable_map_takers_drop(&frame->takers); - if (frame->newest_live && frame->newest.has_value) - std_vec_free_Checked(&frame->newest.value, 1, 1); - if (frame->has_below) - std_vec_free_Checked(&frame->below, 1, 1); - if (frame->base_live) - txn_base_Base_drop(&frame->base); - frame->claimed_live = frame->removals_live = frame->dead_live = frame->took_live = frame->takers_live = false; - frame->newest_live = frame->has_below = frame->base_live = false; } /* @@ -2385,6 +2810,12 @@ typedef struct txn_vacuum_holding_frame_t HostIoIo *io; HostSharedRecord record; uint64 tid; + + /* + * ender is the stored ender whose row the version must hold, or NULL for + * any version. + */ + const uint64 *ender; bool unhold; bool key_live; StdVec key; @@ -2392,6 +2823,8 @@ typedef struct txn_vacuum_holding_frame_t HostDeltaHold hold; bool chain_live; StdVec chain; + bool seen_live; + TxnVisibleSeen seen; bool removed_live; StdVec removed; bool *result; @@ -2400,8 +2833,9 @@ typedef struct txn_vacuum_holding_frame_t } txn_vacuum_holding_frame_t; /* - * txn_vacuum_holding_body finds the version holding the tuple id and, to - * unhold it, takes it out and takes back its count. + * txn_vacuum_holding_body finds the version holding the tuple id, the one + * whose row a stored ender ended when it has one, and, to unhold it, takes it + * out and takes back its count. */ static void txn_vacuum_holding_body(void *frame_) @@ -2427,9 +2861,44 @@ txn_vacuum_holding_body(void *frame_) &frame->chain, frame->error)) return; frame->chain_live = true; - for (size_t i = 0; i < frame->chain.len && version == NULL; i++) - if (txn_vacuum_holds(&STD_VEC_AT(&frame->chain, TxnMemtableVersionVersion, i), frame->tid)) - version = &STD_VEC_AT(&frame->chain, TxnMemtableVersionVersion, i); + + /* + * An ender with OWN is the row's own writer, found by its writer, since + * every rule sees no row of an aborted writer. + */ + if (frame->ender == NULL || (*frame->ender & TXN_BASE_OWN) != 0) + { + for (size_t i = 0; i < frame->chain.len && version == NULL; i++) + { + TxnMemtableVersionVersion *held = &STD_VEC_AT(&frame->chain, TxnMemtableVersionVersion, i); + TxnMemtableVersionRecord stored = txn_memtable_version_Version_record(held); + + if (txn_vacuum_holds(held, frame->tid) && + (frame->ender == NULL || + txn_memtable_version_Record_writer(&stored).xid == (*frame->ender & ~TXN_BASE_OWN))) + version = held; + } + } + else + { + TxnVisibleRule every = {&TXN_FOLD_EVERY_RULE_OPS, NULL}; + + if (!txn_visible_seen(frame->io, every, frame->chain.ptr, frame->chain.len, NULL, &frame->seen, + frame->error)) + return; + frame->seen_live = true; + for (size_t i = 0; i < frame->seen.rows.len && version == NULL; i++) + { + const TxnVisibleSeenRow *row = &STD_VEC_AT(&frame->seen.rows, TxnVisibleSeenRow, i); + TxnMemtableVersionRecord changer; + + if (row->row.put.tuple.f0 != frame->tid || row->changer == NULL) + continue; + changer = txn_memtable_version_Version_record(row->changer); + if (txn_memtable_version_Record_writer(&changer).xid == (*frame->ender & ~TXN_BASE_OWN)) + version = row->row.version; + } + } *frame->result = version != NULL; if (version != NULL && frame->unhold) { @@ -2437,9 +2906,6 @@ txn_vacuum_holding_body(void *frame_) frame->key.len, &version->handle, 1, &frame->removed, frame->error)) return; frame->removed_live = true; - if (frame->removed.len == 1 && STD_VEC_AT(&frame->removed, bool, 0) && - !txn_memtable_table_uncounted(frame->io, version, frame->error)) - return; } frame->ok = true; } @@ -2452,26 +2918,30 @@ txn_vacuum_holding_pad(void *frame_) if (frame->removed_live) std_vec_free_Checked(&frame->removed, sizeof(bool), _Alignof(bool)); + if (frame->seen_live) + txn_visible_Seen_drop(&frame->seen); if (frame->chain_live) txn_memtable_version_versions_drop(&frame->chain, 0); if (frame->hold_live) host_delta_Hold_drop(&frame->hold); if (frame->key_live) std_vec_free_Checked(&frame->key, 1, 1); - frame->removed_live = frame->chain_live = frame->hold_live = frame->key_live = false; + frame->removed_live = frame->seen_live = frame->chain_live = frame->hold_live = frame->key_live = false; } /* * txn_vacuum_holding finds the row version that holds a tuple id, in the - * chain of the key its tuple id table names for it, reporting whether one - * does, and to unhold it takes it out of its key's chain, taking back its - * count at its writer's entry when this removal took it out. + * chain of the key its tuple id table names for it, one whose row a stored + * ender ended when ender isn't NULL, reporting whether one does, and to + * unhold it takes it out of its key's chain, taking back its count at its + * writer's entry when this removal took it out. */ static bool -txn_vacuum_holding(HostIoIo *io, HostSharedRecord record, uint64 tid, bool unhold, bool *result, HostError *error) +txn_vacuum_holding(HostIoIo *io, HostSharedRecord record, uint64 tid, const uint64 *ender, bool unhold, bool *result, + HostError *error) { - txn_vacuum_holding_frame_t frame = {.io = io, .record = record, .tid = tid, .unhold = unhold, .result = result, - .error = error}; + txn_vacuum_holding_frame_t frame = {.io = io, .record = record, .tid = tid, .ender = ender, .unhold = unhold, + .result = result, .error = error}; std_landing(&frame, txn_vacuum_holding_body, txn_vacuum_holding_pad); txn_vacuum_holding_pad(&frame); @@ -2480,14 +2950,21 @@ txn_vacuum_holding(HostIoIo *io, HostSharedRecord record, uint64 tid, bool unhol /* * txn_vacuum_unheld takes out of its key's chain the row version that holds a - * tuple id, if one does, taking back its count. + * tuple id, if one does, taking back its count; with a stored ender, only the + * version of the freed incarnation, whose row that ender ended, so a later + * take of the tuple id a lagging standby already replayed is passed over + * (bugs row 320). An ender of 0 takes out nothing, since every such free's + * incarnation, an aborted row nothing ended, a fold already dropped, on the + * primary and at the standby's same point. */ static bool -txn_vacuum_unheld(HostIoIo *io, HostSharedRecord record, uint64 tid, HostError *error) +txn_vacuum_unheld(HostIoIo *io, HostSharedRecord record, uint64 tid, const uint64 *ender, HostError *error) { bool found; - return txn_vacuum_holding(io, record, tid, true, &found, error); + if (ender != NULL && *ender == 0) + return true; + return txn_vacuum_holding(io, record, tid, ender, true, &found, error); } /* @@ -2497,7 +2974,28 @@ txn_vacuum_unheld(HostIoIo *io, HostSharedRecord record, uint64 tid, HostError * bool txn_vacuum_held(HostIoIo *io, HostSharedRecord record, uint64 tid, bool *result, HostError *error) { - return txn_vacuum_holding(io, record, tid, false, result, error); + return txn_vacuum_holding(io, record, tid, NULL, false, result, error); +} + +/* txn_vacuum_dead_ender returns a claimed dead tuple id's stored ender, field 10's after every freed tuple id's, from the free's dead runs, which hold it. */ +static const uint64 * +txn_vacuum_dead_ender(const TxnRecordTidsFreed *freed, uint64 tid) +{ + size_t at = 0; + + for (size_t i = 0; i < freed->runs.len; i++) + at += STD_VEC_AT(&freed->runs, std_TupleU32U16U16U64_t, i).f2; + for (size_t i = 0; i < freed->dead.len; i++) + { + const std_TupleU32U16U16U64_t *run = &STD_VEC_AT(&freed->dead, std_TupleU32U16U16U64_t, i); + uint64 first = (uint64) run->f0 << 16 | run->f1; + + if (tid >= first && tid < first + run->f2) + return &STD_VEC_AT(&freed->enders, uint64, at + (size_t) (tid - first)); + at += run->f2; + } + Assert(false); + return NULL; } /* txn_vacuum_blocks returns the blocks of a free's runs, each once, in order. */ @@ -2565,18 +3063,24 @@ txn_vacuum_taken_out_body(void *frame_) return; frame->claimed_live = true; for (size_t i = 0; i < frame->claimed.len; i++) - if (!txn_vacuum_unheld(frame->io, record, STD_VEC_AT(&frame->claimed, HostRow, i).f0, frame->error)) + if (!txn_vacuum_unheld(frame->io, record, STD_VEC_AT(&frame->claimed, HostRow, i).f0, + txn_vacuum_dead_ender(frame->freed, STD_VEC_AT(&frame->claimed, HostRow, i).f0), + frame->error)) return; - for (size_t i = 0; i < frame->freed->runs.len; i++) + for (size_t i = 0, ender = 0; i < frame->freed->runs.len; i++) { const std_TupleU32U16U16U64_t *run = &STD_VEC_AT(&frame->freed->runs, std_TupleU32U16U16U64_t, i); uint16 end; if (run->f0 != block) + { + ender += run->f2; continue; + } end = STD_ADD(u16, run->f1, run->f2, 801, 55); - for (uint16 at = run->f1; at < end; at++) - if (!txn_vacuum_unheld(frame->io, record, (uint64) block << 16 | at, frame->error)) + for (uint16 at = run->f1; at < end; at++, ender++) + if (!txn_vacuum_unheld(frame->io, record, (uint64) block << 16 | at, + &STD_VEC_AT(&frame->freed->enders, uint64, ender), frame->error)) return; } frame->claimed_live = false; @@ -2631,9 +3135,11 @@ txn_vacuum_redo_freed_closure_5(void *captures, HostIoIo *io, TxnMemtableTablePr { txn_vacuum_redo_freed_closure_5_captures_t *inside = captures; TxnMemtableVersionWriter none = TXN_MEMTABLE_VERSION_NO_TRANSACTION; + TxnStatusRoomed unroomed = {0}; - return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, none, true, inside->lsn, - inside->has_based, inside->based->ptr, inside->based->len, + return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, none, false, unroomed, true, + inside->lsn, + inside->has_based, inside->based->ptr, inside->based->len, NULL, txn_vacuum_taken_down, &inside->by, NULL, txn_vacuum_nothing, NULL, NULL, result, error); } @@ -2651,16 +3157,13 @@ typedef struct txn_vacuum_redo_frame_t const TxnRecordTidsFreed *freed; const TxnInsertPage *page; uint64 lsn; - bool base_live; - TxnBaseBase base; bool blocks_live; StdVec blocks; bool runs_live; StdVec runs; bool dead_live; StdVec dead; - bool has_below; - StdVec below; + txn_vacuum_Read_t read; bool claims_live; StdVec claims; bool has_based; @@ -2675,39 +3178,35 @@ txn_vacuum_redo_block_drop(txn_vacuum_redo_frame_t *frame) { if (frame->claims_live) std_vec_free_Checked(&frame->claims, TXN_VACUUM_CLAIM); - if (frame->has_below) - std_vec_free_Checked(&frame->below, 1, 1); + txn_vacuum_Read_drop(&frame->read); if (frame->dead_live) std_vec_free_Checked(&frame->dead, TXN_VACUUM_RUN); if (frame->runs_live) std_vec_free_Checked(&frame->runs, TXN_VACUUM_RUN); - frame->claims_live = frame->has_below = frame->dead_live = frame->runs_live = false; + frame->claims_live = frame->dead_live = frame->runs_live = false; } /* * txn_vacuum_redo_body gives each block back unless its entry holds the - * record, takes the values down, then does the take-outs. + * record, reading the base and the block's entry again while an install + * moved the base since they were read, and pausing at + * doltgres-vacuum-freeing after each read of the base; it then takes the + * values down and does the take-outs. */ static void txn_vacuum_redo_body(void *frame_) { txn_vacuum_redo_frame_t *frame = frame_; HostSharedRecord record = host_shared_Held_record(frame->held); - bool some; bool deferring; - if (!txn_state_base_record(frame->io, frame->held, &some, &frame->base, frame->error)) - return; - frame->base_live = some; txn_vacuum_blocks(frame->freed, &frame->blocks); frame->blocks_live = true; for (size_t b = 0; b < frame->blocks.len; b++) { uint32 block = STD_VEC_AT(&frame->blocks, uint32, b); - txn_vacuum_given_closure_2_captures_t captures = {.record = record, .block = block, .below = &frame->below, - .page = frame->page, .freed = &frame->runs, .has_lsn = true, .lsn = frame->lsn}; bool holds; - bool done; + bool done = false; if (!txn_memtable_map_holds(frame->io, record, TXN_MEMTABLE_MAP_MAP_FREE, block, frame->lsn, &holds, frame->error)) @@ -2718,18 +3217,20 @@ txn_vacuum_redo_body(void *frame_) frame->runs_live = true; txn_vacuum_of_block(&frame->freed->dead, block, &frame->dead); frame->dead_live = true; - if (!txn_vacuum_below(frame->io, frame->store, frame->base_live ? &frame->base : NULL, block, - &frame->has_below, &frame->below, frame->error) || - !txn_vacuum_first(frame->has_below, &frame->below, block, frame->base_live ? &frame->base : NULL, - frame->page, &captures.start, frame->error)) - return; - captures.has_below = frame->has_below; - if (!txn_vacuum_given(frame->io, &captures, &frame->dead, &done, &frame->claims, frame->error)) - return; - if (!done) + while (!done) { - *frame->error = txn_vacuum_fork("XX000", "a free's redo was asked to read its values again"); - return; + txn_vacuum_given_closure_2_captures_t captures = {.record = record, .block = block, .read = &frame->read, + .page = frame->page, .freed = &frame->runs, .has_lsn = true, .lsn = frame->lsn}; + + frame->read.io = frame->io; + frame->read.store = frame->store; + frame->read.held = frame->held; + frame->read.block = block; + frame->read.page = frame->page; + if (!txn_vacuum_read(&frame->read, true, frame->error)) + return; + if (!txn_vacuum_given(frame->io, &captures, &frame->dead, &done, &frame->claims, frame->error)) + return; } frame->claims_live = true; txn_vacuum_redo_block_drop(frame); @@ -2794,9 +3295,7 @@ txn_vacuum_redo_pad(void *frame_) std_vec_free_Checked(&frame->based, 1, 1); if (frame->blocks_live) std_vec_free_Checked(&frame->blocks, sizeof(uint32), _Alignof(uint32)); - if (frame->base_live) - txn_base_Base_drop(&frame->base); - frame->has_based = frame->blocks_live = frame->base_live = false; + frame->has_based = frame->blocks_live = false; } /* @@ -2821,3 +3320,3377 @@ txn_vacuum_redo_freed(HostIoIo *io, TreesStoreNodeStore store, const HostSharedH txn_vacuum_redo_pad(&frame); return frame.ok; } + +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/chunks/header.h" +#include "doltgres/chunks/trailer.h" +#include "doltgres/host/delta.h" +#include "doltgres/repo/lib.h" +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" +#include "doltgres/txn/change.h" +#include "doltgres/txn/fold.h" +#include "doltgres/txn/status.h" +#include "doltgres/txn/written.h" + +/* TXN_VACUUM_TESTS_UNWRAP_FAILED is a Result's unwrap's panic on a HostError at a line and column. */ +#define TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, error) \ + STD_RESULT_EXPECT_FAILED((line), (col), "called `Result::unwrap()` on an `Err` value", (error), \ + host_Error_Debug_fmt) + +/* TXN_VACUUM_TESTS_PAGE is PAGE, an 8 kB heap page: its bytes past its header and its most tuples. */ +static const TxnInsertPage TXN_VACUUM_TESTS_PAGE = {.bytes = 8168, .most = 291}; + +/* txn_vacuum_tests_case_t is a case run on a new database's storage, with a reader of its repository. */ +typedef void (*txn_vacuum_tests_case_t) (void *captures, HostIoIo *io, TreesStoreNodeStore store, + const HostSharedHeld *held); + +/* txn_vacuum_tests_entry_t is an Option: a block's newest free space entry, or none. */ +typedef struct txn_vacuum_tests_entry_t +{ + bool has; + TxnFreeSpaceFree free; +} txn_vacuum_tests_entry_t; + +/* + * txn_vacuum_tests_logged_t is Logged: one of a database's records, with its + * end, a row change or a free. + */ +typedef struct txn_vacuum_tests_logged_t +{ + /* freed is whether the record is a free, else a row change. */ + bool freed; + TxnRecordRowChange row; + TxnRecordTidsFreed free; + HostLsn end; +} txn_vacuum_tests_logged_t; + +/* txn_vacuum_tests_every returns a horizon that has passed every xid. */ +SimcTaken +txn_vacuum_tests_every(void) +{ + HostXid horizon = {UINT64_MAX}; + + return simc_non_vacuumable(horizon); +} + +/* txn_vacuum_tests_with runs a case on a new database's storage, with a reader of its repository. */ +static void +txn_vacuum_tests_with(HostIoIo *io, uint32 database, txn_vacuum_tests_case_t case_, void *captures) +{ + HostStorage storage = txn_insert_tests_stored(database); + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_create(io, database, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(832, 91, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!host_shared_read(io, &reading, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(833, 46, &error); + if (!txn_state_state(io, &repo, &reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &held, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(834, 110, &error); + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + case_(captures, io, TREES_STORE_READER_NODE_STORE(&store), &held); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); +} + +/* + * txn_vacuum_tests_inserted inserts a key's row of some heap bytes into a + * database's storage as a transaction's, and returns its tuple id. + */ +static uint64 +txn_vacuum_tests_inserted(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database, + uint64 key, uint64 bytes, HostXid xid) +{ + TxnInsertInsert insert = txn_insert_tests_row(txn_insert_tests_stored(database), key); + TxnInsertRoom room = {.bytes = bytes, .page = TXN_VACUUM_TESTS_PAGE.bytes, .most = TXN_VACUUM_TESTS_PAGE.most}; + HostError error; + uint64 tid; + + insert.writer.xid = xid.f0; + insert.put.heap_bytes = (uint32) bytes; + if (!txn_insert_tests_taken(io, store, held, &insert, &room, NULL, NULL, 0, &tid, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(851, 110, &error); + return tid; +} + +/* txn_vacuum_tests_ended ends a test's transaction, committed or aborted, as its callbacks do. */ +static void +txn_vacuum_tests_ended(HostIoIo *io, HostXid xid, bool committed) +{ + HostError error; + + if (committed) + { + simc_commit(xid); + if (!txn_written_ended(io, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(859, 36, &error); + } + else + { + simc_abort(xid); + if (!txn_written_aborted(io, 0, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(863, 38, &error); + } +} + +/* txn_vacuum_tests_entry returns a block's newest free space entry in a storage's delta, read at the storage's counter. */ +static txn_vacuum_tests_entry_t +txn_vacuum_tests_entry(HostIoIo *io, const HostSharedHeld *held, uint32 block) +{ + HostSharedRecord record = host_shared_Held_record(held); + txn_vacuum_tests_entry_t entry = {0}; + TxnMemtableMapEntry newest; + HostError error; + bool some; + + if (!txn_memtable_map_newest(io, record, TXN_MEMTABLE_MAP_MAP_FREE, block, &some, &newest, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(870, 70, &error); + if (!some || !newest.has_value) + return entry; + if (!txn_free_space_Free_decode(newest.value.ptr, newest.value.len, block, txn_state_counter(record), + TXN_VACUUM_TESTS_PAGE.most, &entry.free, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(871, 84, &error); + std_vec_free_Checked(&newest.value, 1, 1); + entry.has = true; + return entry; +} + +/* txn_vacuum_tests_some returns an entry of a block's free space. */ +static txn_vacuum_tests_entry_t +txn_vacuum_tests_some(TxnFreeSpaceFree free) +{ + return (txn_vacuum_tests_entry_t) + { + true, free + }; +} + +/* txn_vacuum_tests_entry_Debug_fmt is an Option's Debug. */ +static bool +txn_vacuum_tests_entry_Debug_fmt(const void *value, StdFormatter *f) +{ + const txn_vacuum_tests_entry_t *entry = value; + StdOptionRef option = {entry->has, {&entry->free, txn_free_space_Free_Debug_fmt}}; + + return std_option_Debug_fmt(&option, f); +} + +/* txn_vacuum_tests_entry_eq reports whether two Options are equal. */ +static bool +txn_vacuum_tests_entry_eq(const txn_vacuum_tests_entry_t *a, const txn_vacuum_tests_entry_t *b) +{ + return a->has == b->has && (!a->has || txn_free_space_Free_PartialEq_eq(&a->free, &b->free)); +} + +/* txn_vacuum_tests_entry_is asserts at a line and column that a block's entry is the one wanted. */ +static void +txn_vacuum_tests_entry_is(uint32 line, uint32 col, txn_vacuum_tests_entry_t got, txn_vacuum_tests_entry_t want) +{ + STD_ASSERT_EQ(line, col, txn_vacuum_tests_entry_eq(&got, &want), &got, txn_vacuum_tests_entry_Debug_fmt, &want, + txn_vacuum_tests_entry_Debug_fmt); +} + +/* txn_vacuum_tests_logged_free frees a Vec and its records. */ +static void +txn_vacuum_tests_logged_free(StdVec *records) +{ + for (size_t k = 0; k < records->len; k++) + { + txn_vacuum_tests_logged_t *record = &STD_VEC_AT(records, txn_vacuum_tests_logged_t, k); + + if (record->freed) + txn_record_TidsFreed_drop(&record->free); + else + txn_record_RowChange_drop(&record->row); + } + std_vec_free_Checked(records, sizeof(txn_vacuum_tests_logged_t), _Alignof(txn_vacuum_tests_logged_t)); +} + +/* txn_vacuum_tests_logged returns the row changes and frees the simulation logged for a database's storages, in their order. */ +static StdVec +txn_vacuum_tests_logged(uint32 database) +{ + StdVec logged = simc_logged(); + StdVec records; + + std_vec_new(&records, sizeof(txn_vacuum_tests_logged_t), _Alignof(txn_vacuum_tests_logged_t)); + for (size_t k = 0; k < logged.len; k++) + { + simc_TupleU8VecU8Lsn_t *one = &STD_VEC_AT(&logged, simc_TupleU8VecU8Lsn_t, k); + txn_vacuum_tests_logged_t record = {.end = one->f2}; + HostWalKind kind; + HostError error; + + if (host_WalKind_of(one->f0, &kind) && kind == HOST_WAL_KIND_ROW_CHANGE) + { + if (!txn_record_RowChange_decode(one->f1.ptr, one->f1.len, &record.row, &error)) + host_Error_drop(&error); + else if (record.row.storage.database != database) + txn_record_RowChange_drop(&record.row); + else + std_vec_push_Checked(&records, &record, sizeof(record), _Alignof(txn_vacuum_tests_logged_t)); + } + else if (host_WalKind_of(one->f0, &kind) && kind == HOST_WAL_KIND_TIDS_FREED) + { + record.freed = true; + if (!txn_record_TidsFreed_decode(one->f1.ptr, one->f1.len, TXN_VACUUM_TESTS_PAGE.most, + TXN_VACUUM_TESTS_PAGE.bytes, &record.free, &error)) + host_Error_drop(&error); + else if (record.free.storage.database != database) + txn_record_TidsFreed_drop(&record.free); + else + std_vec_push_Checked(&records, &record, sizeof(record), _Alignof(txn_vacuum_tests_logged_t)); + } + std_vec_free_Checked(&one->f1, 1, 1); + } + std_vec_free_Checked(&logged, sizeof(simc_TupleU8VecU8Lsn_t), _Alignof(simc_TupleU8VecU8Lsn_t)); + return records; +} + +/* txn_vacuum_tests_freed_Debug_fmt is a Freed's Debug, the tuple (tid, bytes, horizon, values). */ +static bool +txn_vacuum_tests_freed_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnVacuumFreed *freed = value; + StdTupleRef tuple = STD_TUPLE(4, STD_ITEM(&freed->f0, std_u64_Debug_fmt), STD_ITEM(&freed->f1, std_u64_Debug_fmt), + STD_ITEM(&freed->f2, std_u64_Debug_fmt), + STD_ITEM(&freed->f3, chunks_header_VecU32_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_vacuum_tests_found_Debug_fmt is a Vec's Debug. */ +static bool +txn_vacuum_tests_found_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *found = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < found->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(found, TxnVacuumFreed, k), txn_vacuum_tests_freed_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_vacuum_tests_found_free frees a Vec and each one's value ids. */ +static void +txn_vacuum_tests_found_free(StdVec *found) +{ + for (size_t k = 0; k < found->len; k++) + std_vec_free_Checked(&STD_VEC_AT(found, TxnVacuumFreed, k).f3, sizeof(uint32), _Alignof(uint32)); + std_vec_free_Checked(found, sizeof(TxnVacuumFreed), _Alignof(TxnVacuumFreed)); +} + +/* + * txn_vacuum_tests_found_is_with asserts at a line and column, with a + * message or none, that the tuple ids found are those wanted, none naming a + * value. + */ +static void +txn_vacuum_tests_found_is_with(uint32 line, uint32 col, const StdVec *found, const TxnVacuumFreed *want, + size_t want_len, const StdFmtArguments *message) +{ + StdVec wanted = {(void *) want, want_len, want_len}; + bool eq = found->len == want_len; + + for (size_t k = 0; eq && k < want_len; k++) + { + const TxnVacuumFreed *got = &STD_VEC_AT(found, TxnVacuumFreed, k); + + eq = got->f0 == want[k].f0 && got->f1 == want[k].f1 && got->f2 == want[k].f2 && got->f3.len == 0; + } + if (!eq) + std_assert_failed(RUST_FILE, line, col, "==", found, txn_vacuum_tests_found_Debug_fmt, &wanted, + txn_vacuum_tests_found_Debug_fmt, message); +} + +/* txn_vacuum_tests_found_is asserts at a line and column that the tuple ids found are those wanted, none naming a value. */ +static void +txn_vacuum_tests_found_is(uint32 line, uint32 col, const StdVec *found, const TxnVacuumFreed *want, size_t want_len) +{ + txn_vacuum_tests_found_is_with(line, col, found, want, want_len, NULL); +} + +/* txn_vacuum_tests_found returns the tuple ids never_held finds under a horizon, unwrapped at a line and column. */ +static StdVec +txn_vacuum_tests_found(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, const SimcTaken *horizon, + uint32 line, uint32 col) +{ + HostSnapshot lent = simc_Taken_lent(horizon); + HostError error; + StdVec found; + + if (!txn_vacuum_never_held(io, store, held, &TXN_VACUUM_TESTS_PAGE, &lent, &found, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); + return found; +} + +/* txn_vacuum_tests_found_every returns the tuple ids never_held finds past every xid, unwrapped at a line and column. */ +static StdVec +txn_vacuum_tests_found_every(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 line, + uint32 col) +{ + SimcTaken every = txn_vacuum_tests_every(); + StdVec found = txn_vacuum_tests_found(io, store, held, &every, line, col); + + simc_Taken_drop(&every); + return found; +} + +/* + * txn_vacuum_tests_claimed frees a test storage's tuple ids, or for dead + * gives back their bytes, as a logged storage with no TOAST storage, and + * returns the dead tuple ids claimed, unwrapped at a line and column. + */ +static StdVec +txn_vacuum_tests_claimed(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database, + const TxnVacuumFreed *tids, size_t tids_len, bool dead, uint32 line, uint32 col) +{ + HostStorage toast = {0}; + HostError error; + StdVec claimed; + + if (!txn_vacuum_freed(io, store, held, NULL, toast, txn_insert_tests_stored(database), true, + &TXN_VACUUM_TESTS_PAGE, tids, tids_len, dead, &claimed, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); + return claimed; +} + +/* txn_vacuum_tests_freed frees a test storage's found tuple ids as claimed does, and drops what it claimed. */ +static void +txn_vacuum_tests_freed(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database, + const StdVec *found, bool dead, uint32 line, uint32 col) +{ + StdVec claimed = txn_vacuum_tests_claimed(io, store, held, database, found->ptr, found->len, dead, line, col); + + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); +} + +/* txn_vacuum_tests_redone redoes a free on a storage, unwrapped at a line and column. */ +static void +txn_vacuum_tests_redone(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, + const TxnRecordTidsFreed *free, HostLsn end, uint32 line, uint32 col) +{ + HostError error; + + if (!txn_vacuum_redo_freed(io, store, held, NULL, free, &TXN_VACUUM_TESTS_PAGE, end, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); +} + +/* txn_vacuum_tests_redo redoes a row change on a storage, unwrapped at a line and column. */ +static void +txn_vacuum_tests_redo(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, + const TxnRecordRowChange *change, HostLsn end, uint32 line, uint32 col) +{ + HostError error; + + if (!txn_insert_redo(io, store, held, change, end, &TXN_VACUUM_TESTS_PAGE, NULL, NULL, 0, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); +} + +/* + * txn_vacuum_tests_aborted_t is the aborted insert case's captures: the + * entry the primary's free left, the records logged, and the free's place + * among them. + */ +typedef struct txn_vacuum_tests_aborted_t +{ + txn_vacuum_tests_entry_t primary; + StdVec records; + size_t at; +} txn_vacuum_tests_aborted_t; + +/* txn_vacuum_tests_aborted_primary is the aborted insert case's primary: it frees the aborted row's tuple id and takes it again. */ +static void +txn_vacuum_tests_aborted_primary(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed want[] = {{2, 300, 0}}; + static const uint16 two[] = {2}; + txn_vacuum_tests_aborted_t *c = captures; + HostXid one = simc_begin(); + HostXid second; + HostXid three; + HostXid four; + StdVec found; + + STD_ASSERT_EQ_U64(910, 13, txn_vacuum_tests_inserted(io, store, held, 9428, 1, 100, one), 1); + txn_vacuum_tests_ended(io, one, true); + second = simc_begin(); + STD_ASSERT_EQ_U64(913, 13, txn_vacuum_tests_inserted(io, store, held, 9428, 2, 300, second), 2); + txn_vacuum_tests_ended(io, second, false); + three = simc_begin(); + STD_ASSERT_EQ_U64(916, 13, txn_vacuum_tests_inserted(io, store, held, 9428, 3, 100, three), 3); + txn_vacuum_tests_ended(io, three, true); + found = txn_vacuum_tests_found_every(io, store, held, 918, 79); + txn_vacuum_tests_found_is(919, 13, &found, want, lengthof(want)); + txn_vacuum_tests_freed(io, store, held, 9428, &found, false, 920, 90); + txn_vacuum_tests_entry_is(921, 13, txn_vacuum_tests_entry(io, held, 0), + txn_vacuum_tests_some(txn_free_space_Free_new(8168 - 500 + 300, two, 1, 291))); + txn_vacuum_tests_found_free(&found); + found = txn_vacuum_tests_found_every(io, store, held, 922, 78); + txn_vacuum_tests_found_is(922, 13, &found, NULL, 0); + txn_vacuum_tests_found_free(&found); + c->primary = txn_vacuum_tests_entry(io, held, 0); + four = simc_begin(); + STD_ASSERT_EQ_U64(925, 13, txn_vacuum_tests_inserted(io, store, held, 9428, 4, 300, four), 2); + txn_vacuum_tests_ended(io, four, true); +} + +/* txn_vacuum_tests_aborted_replica is the aborted insert case's replica: it redoes the primary's records up to the free, then the free twice. */ +static void +txn_vacuum_tests_aborted_replica(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + txn_vacuum_tests_aborted_t *c = captures; + const txn_vacuum_tests_logged_t *free = &STD_VEC_AT(&c->records, txn_vacuum_tests_logged_t, c->at); + + for (size_t k = 0; k < c->at; k++) + { + const txn_vacuum_tests_logged_t *record = &STD_VEC_AT(&c->records, txn_vacuum_tests_logged_t, k); + + if (!record->freed) + txn_vacuum_tests_redo(io, store, held, &record->row, record->end, 936, 78); + } + txn_vacuum_tests_redone(io, store, held, &free->free, free->end, 939, 68); + txn_vacuum_tests_entry_is(940, 13, txn_vacuum_tests_entry(io, held, 0), c->primary); + txn_vacuum_tests_redone(io, store, held, &free->free, free->end, 941, 68); + txn_vacuum_tests_entry_is(942, 13, txn_vacuum_tests_entry(io, held, 0), c->primary); +} + +/* txn_vacuum_tests_freed_is asserts at a line and column that a logged free is the one wanted. */ +static void +txn_vacuum_tests_freed_is(uint32 line, uint32 col, const TxnRecordTidsFreed *got, const TxnRecordTidsFreed *want) +{ + STD_ASSERT_EQ(line, col, txn_record_TidsFreed_PartialEq_eq(got, want), got, txn_record_TidsFreed_Debug_fmt, want, + txn_record_TidsFreed_Debug_fmt); +} + +/* + * Bugs row 44's first case and D4's standby case: an aborted insert on the + * fresh page leaves its tuple id taken with no row, which VACUUM frees with + * the 300 heap bytes its own take lowered, logging one TidsFreed with no + * horizon, after which the next insert of its size takes it again, while the + * committed rows' tuple ids stay. A storage that redoes the primary's records + * up to the free holds the same free space entry, and a free redone again + * gives nothing back twice. + */ +void +test_txn_vacuum_tests_an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again(void) +{ + static std_TupleU32U16U16U64_t runs[] = {{0, 2, 1, 300}}; + static uint64 enders[] = {0}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_aborted_t c = {0}; + TxnRecordTidsFreed want = {.storage = txn_insert_tests_stored(9428), .runs = {runs, 1, 1}, .enders = {enders, 1, 1}}; + + txn_vacuum_tests_with(io, 9428, txn_vacuum_tests_aborted_primary, &c); + c.records = txn_vacuum_tests_logged(9428); + while (c.at < c.records.len && !STD_VEC_AT(&c.records, txn_vacuum_tests_logged_t, c.at).freed) + c.at++; + if (c.at == c.records.len) + STD_OPTION_UNWRAP_FAILED(929, 88); + txn_vacuum_tests_freed_is(932, 9, &STD_VEC_AT(&c.records, txn_vacuum_tests_logged_t, c.at).free, &want); + txn_vacuum_tests_with(io, 9432, txn_vacuum_tests_aborted_replica, &c); + txn_vacuum_tests_logged_free(&c.records); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_takers_free frees a Vec and each taker's entry and value ids. */ +static void +txn_vacuum_tests_takers_free(StdVec *takers) +{ + for (size_t k = 0; k < takers->len; k++) + { + txn_TupleWriterRowEntryU64TupleVecU32Bool_t *taker = &STD_VEC_AT(takers, + txn_TupleWriterRowEntryU64TupleVecU32Bool_t, k); + + if (taker->f2.has_value) + std_vec_free_Checked(&taker->f2.value, 1, 1); + std_vec_free_Checked(&taker->f4.f0, sizeof(uint32), _Alignof(uint32)); + } + std_vec_free_Checked(takers, sizeof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t), + _Alignof(txn_TupleWriterRowEntryU64TupleVecU32Bool_t)); +} + +/* txn_vacuum_tests_takers returns the takers of a block's free space entry in a storage's delta, unwrapped at a line and column. */ +static StdVec +txn_vacuum_tests_takers(HostIoIo *io, const HostSharedHeld *held, uint32 line, uint32 col) +{ + HostError error; + StdVec takers; + + if (!txn_memtable_map_takers(io, host_shared_Held_record(held), TXN_MEMTABLE_MAP_MAP_FREE, 0, &takers, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); + return takers; +} + +/* txn_vacuum_tests_u64s_is asserts at a line and column that some u64s are those wanted, as arrays compare. */ +static void +txn_vacuum_tests_u64s_is(uint32 line, uint32 col, const uint64 *got, const uint64 *want, size_t len) +{ + StdVec left = {(void *) got, len, len}; + StdVec right = {(void *) want, len, len}; + + STD_ASSERT_EQ(line, col, memcmp(got, want, len * sizeof(uint64)) == 0, &left, chunks_trailer_VecU64_Debug_fmt, + &right, chunks_trailer_VecU64_Debug_fmt); +} + +/* txn_vacuum_tests_emptying is the emptying take's case: an aborted insert's take empties the fresh page's entry, and its free gives back all its bytes. */ +static void +txn_vacuum_tests_emptying(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed want[] = {{2, 4084, 0}}; + static const uint16 two[] = {2}; + HostXid one = simc_begin(); + HostXid second; + HostXid three; + StdVec takers; + StdVec found; + + STD_ASSERT_EQ_U64(955, 13, txn_vacuum_tests_inserted(io, store, held, 9429, 1, 4084, one), 1); + txn_vacuum_tests_ended(io, one, true); + second = simc_begin(); + STD_ASSERT_EQ_U64(958, 13, txn_vacuum_tests_inserted(io, store, held, 9429, 2, 4084, second), 2); + txn_vacuum_tests_ended(io, second, false); + takers = txn_vacuum_tests_takers(io, held, 960, 73); + { + const txn_TupleWriterRowEntryU64TupleVecU32Bool_t *last = + takers.len == 0 ? NULL : &STD_VEC_AT(&takers, txn_TupleWriterRowEntryU64TupleVecU32Bool_t, takers.len - 1); + HostRow row = {2}; + StdBytes value = last == NULL ? (StdBytes) {0} : (StdBytes) {last->f2.value.ptr, last->f2.value.len}; + StdOptionRef entry = {last != NULL && last->f2.has_value, {&value, std_bytes_Debug_fmt}}; + StdOptionRef none = {false, {NULL, std_bytes_Debug_fmt}}; + StdTupleRef got = STD_TUPLE(3, STD_ITEM(last == NULL ? NULL : &last->f0.xid, std_u64_Debug_fmt), + STD_ITEM(last == NULL ? NULL : &last->f1, host_Row_Debug_fmt), + STD_ITEM(&entry, std_option_Debug_fmt)); + StdTupleRef wanted = STD_TUPLE(3, STD_ITEM(&second.f0, std_u64_Debug_fmt), STD_ITEM(&row, host_Row_Debug_fmt), + STD_ITEM(&none, std_option_Debug_fmt)); + StdOptionRef left = {last != NULL, {&got, std_tuple_Debug_fmt}}; + StdOptionRef right = {true, {&wanted, std_tuple_Debug_fmt}}; + + STD_ASSERT_EQ(962, 13, last != NULL && last->f0.xid == second.f0 && last->f1.f0 == 2 && + !last->f2.has_value, &left, std_option_Debug_fmt, &right, std_option_Debug_fmt); + } + txn_vacuum_tests_takers_free(&takers); + found = txn_vacuum_tests_found_every(io, store, held, 963, 79); + txn_vacuum_tests_found_is(964, 13, &found, want, lengthof(want)); + txn_vacuum_tests_freed(io, store, held, 9429, &found, false, 965, 90); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_entry_is(966, 13, txn_vacuum_tests_entry(io, held, 0), + txn_vacuum_tests_some(txn_free_space_Free_new(4084, two, 1, 291))); + three = simc_begin(); + STD_ASSERT_EQ_U64(968, 13, txn_vacuum_tests_inserted(io, store, held, 9429, 3, 4084, three), 2); + txn_vacuum_tests_ended(io, three, true); +} + +/* + * P1's emptying take: an aborted insert whose take leaves the fresh page no + * bytes removes the page's entry by an Emptied holding its tuple id, and + * VACUUM frees that tuple id with all 4084 bytes the entry held before it, so + * the next insert of its size takes it again. + */ +void +test_txn_vacuum_tests_a_take_that_empties_its_entry_is_freed_with_all_the_bytes_the_entry_held(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9429, txn_vacuum_tests_emptying, NULL); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_unnamed is the unnamed take's case: a spent place stays taken while its taker runs, and is freed once it ended. */ +static void +txn_vacuum_tests_unnamed(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed want[] = {{1, 120, 0}}; + HostXid spender = simc_begin(); + TxnRecordSpent spent = {.writer = {.xid = spender.f0}, .storage = txn_insert_tests_stored(9430), .tuple = {1}, + .heap_bytes = 120}; + HostXid next; + HostError error; + StdVec found; + + if (!txn_insert_redo_spent(io, store, held, &spent, &TXN_VACUUM_TESTS_PAGE, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(984, 64, &error); + found = txn_vacuum_tests_found_every(io, store, held, 985, 78); + txn_vacuum_tests_found_is(985, 13, &found, NULL, 0); + txn_vacuum_tests_found_free(&found); + simc_commit(spender); + found = txn_vacuum_tests_found_every(io, store, held, 987, 79); + txn_vacuum_tests_found_is(988, 13, &found, want, lengthof(want)); + txn_vacuum_tests_freed(io, store, held, 9430, &found, false, 989, 90); + txn_vacuum_tests_found_free(&found); + next = simc_begin(); + STD_ASSERT_EQ_U64(991, 13, txn_vacuum_tests_inserted(io, store, held, 9430, 1, 120, next), 1); + txn_vacuum_tests_ended(io, next, true); +} + +/* + * D2's race, bugs row 73's spent place, and bugs row 44's insert failed after + * its take and its value id Retry after its take, each of which leaves a + * take that no row names: it stays taken while its taker's transaction runs, + * and VACUUM frees it with its bytes once that transaction ended, after which + * the next insert takes it. + */ +void +test_txn_vacuum_tests_a_take_no_row_names_is_freed_only_once_its_taker_has_ended(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9430, txn_vacuum_tests_unnamed, NULL); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_leaving_t is the aborted leave case's captures: the store, the state, and the block the last insert took. */ +typedef struct txn_vacuum_tests_leaving_t +{ + TreesStoreNodeStore store; + const HostSharedHeld *held; + bool has_target; + uint32 target; +} txn_vacuum_tests_leaving_t; + +/* txn_vacuum_tests_leaving_insert is the aborted leave case's insert: a key's row of some heap bytes at the last insert's block, committed or aborted. */ +static uint64 +txn_vacuum_tests_leaving_insert(txn_vacuum_tests_leaving_t *c, HostIoIo *io, uint64 key, uint64 bytes, + bool committed) +{ + HostXid writer = simc_begin(); + TxnInsertInsert row = txn_insert_tests_row(txn_insert_tests_stored(9435), key); + TxnInsertRoom room = {.bytes = bytes, .page = TXN_VACUUM_TESTS_PAGE.bytes, .most = TXN_VACUUM_TESTS_PAGE.most}; + HostError error; + uint64 tid; + + row.writer.xid = writer.f0; + row.has_target = c->has_target; + row.target = c->target; + if (!txn_insert_tests_taken(io, c->store, c->held, &row, &room, NULL, NULL, 0, &tid, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1011, 103, &error); + c->has_target = true; + c->target = (uint32) (tid >> 16); + txn_vacuum_tests_ended(io, writer, committed); + return tid; +} + +/* txn_vacuum_tests_leaving is the aborted leave case: the new page's tuple id an aborted insert took is freed and taken again, then the old page's high offset. */ +static void +txn_vacuum_tests_leaving(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed want[] = {{1 << 16 | 1, 3200, 0}}; + static const uint64 first[] = {1, 2}; + static const uint64 again[] = {1 << 16 | 1, 1 << 16 | 2, 3}; + txn_vacuum_tests_leaving_t c = {.store = store, .held = held}; + uint64 taken[3]; + StdVec found; + + taken[0] = txn_vacuum_tests_leaving_insert(&c, io, 1, 2500, true); + taken[1] = txn_vacuum_tests_leaving_insert(&c, io, 2, 2500, true); + txn_vacuum_tests_u64s_is(1016, 13, taken, first, 2); + STD_ASSERT_EQ_U64(1017, 13, txn_vacuum_tests_leaving_insert(&c, io, 3, 3200, false), 1 << 16 | 1); + found = txn_vacuum_tests_found_every(io, store, held, 1018, 79); + txn_vacuum_tests_found_is(1019, 13, &found, want, lengthof(want)); + txn_vacuum_tests_freed(io, store, held, 9435, &found, false, 1020, 90); + txn_vacuum_tests_found_free(&found); + taken[0] = txn_vacuum_tests_leaving_insert(&c, io, 4, 3000, true); + taken[1] = txn_vacuum_tests_leaving_insert(&c, io, 5, 3000, true); + taken[2] = txn_vacuum_tests_leaving_insert(&c, io, 6, 3000, true); + txn_vacuum_tests_u64s_is(1022, 13, taken, again, 3); +} + +/* + * Bugs row 44's aborted leave: an aborted insert that left the fresh page for + * a new one keeps the leave, and VACUUM frees the new page's tuple id it + * took, so the next row, preferring the backend's last block, takes (1,1) + * again, and once the new page is full a row the old page can hold takes its + * high offset (0,3). + */ +void +test_txn_vacuum_tests_an_aborted_leaves_old_offsets_and_its_new_tuple_id_are_handed_out_again(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9435, txn_vacuum_tests_leaving, NULL); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_skipped_t is the skipped index pass case's captures: the entries the primary's frees left, and the records logged. */ +typedef struct txn_vacuum_tests_skipped_t +{ + txn_vacuum_tests_entry_t primary[2]; + StdVec records; +} txn_vacuum_tests_skipped_t; + +/* txn_vacuum_tests_skipped_primary is the skipped index pass case's primary: a dead tuple id's bytes come back, then its offset. */ +static void +txn_vacuum_tests_skipped_primary(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed want[] = {{2, 300, 0}}; + static const TxnVacuumFreed drained[] = {{2, 0, 0}}; + static const uint16 two[] = {2}; + txn_vacuum_tests_skipped_t *c = captures; + HostXid one = simc_begin(); + HostXid second; + HostXid three; + HostXid four; + HostXid five; + StdVec found; + StdVec dead; + + STD_ASSERT_EQ_U64(1039, 13, txn_vacuum_tests_inserted(io, store, held, 9431, 1, 100, one), 1); + txn_vacuum_tests_ended(io, one, true); + second = simc_begin(); + STD_ASSERT_EQ_U64(1042, 13, txn_vacuum_tests_inserted(io, store, held, 9431, 2, 300, second), 2); + txn_vacuum_tests_ended(io, second, false); + three = simc_begin(); + STD_ASSERT_EQ_U64(1045, 13, txn_vacuum_tests_inserted(io, store, held, 9431, 3, 100, three), 3); + txn_vacuum_tests_ended(io, three, true); + found = txn_vacuum_tests_found_every(io, store, held, 1047, 79); + txn_vacuum_tests_found_is(1048, 13, &found, want, lengthof(want)); + txn_vacuum_tests_freed(io, store, held, 9431, &found, true, 1049, 89); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_entry_is(1050, 13, txn_vacuum_tests_entry(io, held, 0), + txn_vacuum_tests_some(txn_free_space_Free_new(8168 - 500 + 300, NULL, 0, 291))); + c->primary[0] = txn_vacuum_tests_entry(io, held, 0); + dead = txn_vacuum_tests_found_every(io, store, held, 1052, 78); + txn_vacuum_tests_found_is(1053, 13, &dead, drained, lengthof(drained)); + txn_vacuum_tests_freed(io, store, held, 9431, &dead, true, 1054, 88); + txn_vacuum_tests_found_free(&dead); + txn_vacuum_tests_entry_is(1055, 13, txn_vacuum_tests_entry(io, held, 0), c->primary[0]); + four = simc_begin(); + STD_ASSERT_EQ_U64(1057, 13, txn_vacuum_tests_inserted(io, store, held, 9431, 4, 300, four), 4); + txn_vacuum_tests_ended(io, four, true); + found = txn_vacuum_tests_found_every(io, store, held, 1059, 79); + txn_vacuum_tests_found_is(1060, 13, &found, drained, lengthof(drained)); + txn_vacuum_tests_freed(io, store, held, 9431, &found, false, 1061, 90); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_entry_is(1062, 13, txn_vacuum_tests_entry(io, held, 0), + txn_vacuum_tests_some(txn_free_space_Free_new(8168 - 800 + 300, two, 1, 291))); + c->primary[1] = txn_vacuum_tests_entry(io, held, 0); + five = simc_begin(); + STD_ASSERT_EQ_U64(1065, 13, txn_vacuum_tests_inserted(io, store, held, 9431, 5, 100, five), 2); + txn_vacuum_tests_ended(io, five, true); +} + +/* txn_vacuum_tests_entries_Debug_fmt is a Vec>'s Debug. */ +static bool +txn_vacuum_tests_entries_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *entries = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < entries->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(entries, txn_vacuum_tests_entry_t, k), txn_vacuum_tests_entry_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_vacuum_tests_skipped_replica is the skipped index pass case's replica: it redoes the primary's records before key 5's row, each free twice. */ +static void +txn_vacuum_tests_skipped_replica(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const uint8 five[8] = {0, 0, 0, 0, 0, 0, 0, 5}; + txn_vacuum_tests_skipped_t *c = captures; + StdVec primary = {c->primary, lengthof(c->primary), lengthof(c->primary)}; + StdVec redone; + bool eq; + + std_vec_new(&redone, sizeof(txn_vacuum_tests_entry_t), _Alignof(txn_vacuum_tests_entry_t)); + for (size_t k = 0; k < c->records.len; k++) + { + const txn_vacuum_tests_logged_t *record = &STD_VEC_AT(&c->records, txn_vacuum_tests_logged_t, k); + txn_vacuum_tests_entry_t once; + + if (!record->freed && record->row.key.len == sizeof(five) && + memcmp(record->row.key.ptr, five, sizeof(five)) == 0) + break; + if (!record->freed) + { + txn_vacuum_tests_redo(io, store, held, &record->row, record->end, 1085, 106); + continue; + } + txn_vacuum_tests_redone(io, store, held, &record->free, record->end, 1087, 81); + once = txn_vacuum_tests_entry(io, held, 0); + txn_vacuum_tests_redone(io, store, held, &record->free, record->end, 1089, 81); + txn_vacuum_tests_entry_is(1090, 25, txn_vacuum_tests_entry(io, held, 0), once); + std_vec_push_Checked(&redone, &once, sizeof(once), _Alignof(txn_vacuum_tests_entry_t)); + } + eq = redone.len == primary.len; + for (size_t k = 0; eq && k < redone.len; k++) + eq = txn_vacuum_tests_entry_eq(&STD_VEC_AT(&redone, txn_vacuum_tests_entry_t, k), &c->primary[k]); + STD_ASSERT_EQ(1095, 13, eq, &redone, txn_vacuum_tests_entries_Debug_fmt, &primary, + txn_vacuum_tests_entries_Debug_fmt); + std_vec_free_Checked(&redone, sizeof(txn_vacuum_tests_entry_t), _Alignof(txn_vacuum_tests_entry_t)); +} + +/* txn_vacuum_tests_frees_Debug_fmt is a Vec<&TidsFreed>'s Debug. */ +static bool +txn_vacuum_tests_frees_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *frees = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < frees->len; k++) + std_DebugList_entry(&list, STD_VEC_AT(frees, const TxnRecordTidsFreed *, k), txn_record_TidsFreed_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_vacuum_tests_frees_is asserts at a line and column that the frees logged are those wanted, in their order. */ +static void +txn_vacuum_tests_frees_is(uint32 line, uint32 col, const StdVec *records, const TxnRecordTidsFreed *const *want, + size_t want_len) +{ + StdVec got; + StdVec wanted = {(void *) want, want_len, want_len}; + bool eq; + + std_vec_new(&got, sizeof(const TxnRecordTidsFreed *), _Alignof(const TxnRecordTidsFreed *)); + for (size_t k = 0; k < records->len; k++) + { + const txn_vacuum_tests_logged_t *record = &STD_VEC_AT(records, txn_vacuum_tests_logged_t, k); + const TxnRecordTidsFreed *free = &record->free; + + if (record->freed) + std_vec_push_Checked(&got, &free, sizeof(free), _Alignof(const TxnRecordTidsFreed *)); + } + eq = got.len == want_len; + for (size_t k = 0; eq && k < want_len; k++) + eq = txn_record_TidsFreed_PartialEq_eq(STD_VEC_AT(&got, const TxnRecordTidsFreed *, k), want[k]); + STD_ASSERT_EQ(line, col, eq, &got, txn_vacuum_tests_frees_Debug_fmt, &wanted, txn_vacuum_tests_frees_Debug_fmt); + std_vec_free_Checked(&got, sizeof(const TxnRecordTidsFreed *), _Alignof(const TxnRecordTidsFreed *)); +} + +/* + * D5: a VACUUM that skips its index pass gives back an aborted insert's 300 + * bytes and keeps its offset taken, as heap's prune leaves an LP_DEAD line + * pointer with its space reclaimed, logging them as field 6's runs, so the + * next row takes the counter's (0,4) in that space; a later VACUUM finds the + * tuple id with no bytes left and frees its offset alone, and the next row + * takes (0,2). A storage that redoes the primary's records holds the same + * entry after each free, and each free redone again gives nothing back twice. + */ +void +test_txn_vacuum_tests_a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone(void) +{ + static std_TupleU32U16U16U64_t dead_runs[] = {{0, 2, 1, 300}}; + static std_TupleU32U16U16U64_t after_runs[] = {{0, 2, 1, 0}}; + static uint64 enders[] = {0}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_skipped_t c = {0}; + TxnRecordTidsFreed dead = {.storage = txn_insert_tests_stored(9431), .dead = {dead_runs, 1, 1}, .enders = {enders, 1, 1}}; + TxnRecordTidsFreed after = {.storage = dead.storage, .runs = {after_runs, 1, 1}, .enders = {enders, 1, 1}}; + const TxnRecordTidsFreed *const want[] = {&dead, &after}; + + txn_vacuum_tests_with(io, 9431, txn_vacuum_tests_skipped_primary, &c); + c.records = txn_vacuum_tests_logged(9431); + txn_vacuum_tests_frees_is(1079, 9, &c.records, want, lengthof(want)); + txn_vacuum_tests_with(io, 9433, txn_vacuum_tests_skipped_replica, &c); + txn_vacuum_tests_logged_free(&c.records); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_full_block is the full block case: a dead tuple id on a block whose entry has no free offset keeps its bytes until its free. */ +static void +txn_vacuum_tests_full_block(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed want[] = {{5, 28, 0}}; + static const uint16 five[] = {5}; + txn_vacuum_tests_entry_t none = {0}; + StdVec found; + + for (uint64 key = 1; key <= 292; key++) + { + HostXid writer = simc_begin(); + uint64 tid = txn_vacuum_tests_inserted(io, store, held, 9434, key, 28, writer); + + STD_ASSERT_EQ_U64(1111, 17, tid, key <= 291 ? key : 1 << 16 | 1); + txn_vacuum_tests_ended(io, writer, key != 5); + } + txn_vacuum_tests_entry_is(1114, 13, txn_vacuum_tests_entry(io, held, 0), none); + found = txn_vacuum_tests_found_every(io, store, held, 1115, 79); + txn_vacuum_tests_found_is(1116, 13, &found, want, lengthof(want)); + txn_vacuum_tests_freed(io, store, held, 9434, &found, true, 1117, 89); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_entry_is(1118, 13, txn_vacuum_tests_entry(io, held, 0), none); + found = txn_vacuum_tests_found_every(io, store, held, 1119, 79); + txn_vacuum_tests_found_is(1120, 13, &found, want, lengthof(want)); + txn_vacuum_tests_freed(io, store, held, 9434, &found, false, 1121, 90); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_entry_is(1122, 13, txn_vacuum_tests_entry(io, held, 0), + txn_vacuum_tests_some(txn_free_space_Free_new(28, five, 1, 291))); +} + +/* + * D5's kept bytes: a dead tuple id on a block other than the fresh page whose + * entry has no free offset left takes no marker, so it keeps its 28 bytes, + * and the later free gives back its offset with them; heap's + * PageGetHeapFreeSpace likewise gives such a page no room while its pointers + * are all used, dead ones included. + */ +void +test_txn_vacuum_tests_a_dead_tuple_id_on_a_full_block_keeps_its_bytes_until_its_free(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9434, txn_vacuum_tests_full_block, NULL); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_updated_decide is updated's read of the replaced key's versions: it goes on as the transaction's command, over the newest. */ +static bool +txn_vacuum_tests_updated_decide(void *captures, HostIoIo *io, TxnMemtableVersionVersion *versions, size_t versions_len, + TxnChangeRead *result, void *stopped, HostError *error) +{ + result->tag = TXN_CHANGE_READ_GO; + result->u.go.shown.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND; + result->u.go.cmin = 0; + result->u.go.has_row = versions_len > 0; + if (versions_len > 0) + result->u.go.row = versions[0].handle; + return true; +} + +/* + * txn_vacuum_tests_updated updates a key's row of a database's storage to + * some heap bytes as a transaction's, taking a block by heap's order after a + * take-time prune when prune names the snapshots, and returns the new row's + * tuple id. + */ +static uint64 +txn_vacuum_tests_updated(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database, + uint64 key, uint64 tuple, uint64 bytes, HostXid xid, const TxnVacuumPrune *prune) +{ + static const StdFmtPiece pieces[] = {STD_FMT_LIT("a case's update gave "), STD_FMT_ARG(0)}; + HostIoWaits waits = host_io_Waits_testing(); + TxnInsertInsert row = txn_insert_tests_row(txn_insert_tests_stored(database), key); + uint32 stripe = txn_change_stripe(&row.storage, row.key.ptr, row.key.len, true); + TxnInsertReplaces replaces = {.tuple = {tuple}, .lock = TXN_MEMTABLE_LOCK_MODE_NO_KEY_UPDATE, .stripe = stripe}; + TxnInsertKeyed keyed = {.stripe = stripe}; + TxnInsertRoom room = {.bytes = bytes, .page = TXN_VACUUM_TESTS_PAGE.bytes, .most = TXN_VACUUM_TESTS_PAGE.most}; + TxnInsertInserted inserted; + TxnInsertInsertedRef other = {&inserted, {NULL, std_unit_Debug_fmt}}; + StdFmtArg args[] = {{&other, txn_insert_Inserted_Debug_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + HostError error; + + row.writer.xid = xid.f0; + std_vec_clone(&replaces.key, &row.key, 1, 1); + if (!txn_insert_update(io, store, held, &row, &replaces, &room, prune, NULL, NULL, 0, keyed, &waits, + txn_vacuum_tests_updated_decide, NULL, NULL, NULL, &inserted, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1152, 111, &error); + if (inserted.tag != TXN_INSERT_INSERTED_TOOK) + std_panic_fmt(RUST_FILE, 1154, 22, &arguments); + return inserted.u.took.tuple; +} + +/* txn_vacuum_tests_committed_status takes a transaction's commit status entry, its commit record ending at an LSN, unwrapped at a line and column. */ +static void +txn_vacuum_tests_committed_status(HostIoIo *io, HostXid xid, HostLsn end, uint32 line, uint32 col) +{ + TxnStatusOutcome outcome = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = end}; + TxnStatusRoomed roomed; + HostError error; + bool took; + + if (!txn_status_take(io, xid, outcome, &roomed, &took, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); +} + +/* + * txn_vacuum_tests_filled fills a database's storage with fourteen committed + * rows of 1036 heap bytes, seven a page, (0,1) to (1,7), and commits an + * update of key 1 whose new row finds no room on block 0 and goes to (2,1), + * setting block 0's full flag, and returns the updater's xid. When committed + * is set, each commit first takes its status entry there, which a fold reads. + */ +static HostXid +txn_vacuum_tests_filled(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database, + HostLsn committed) +{ + HostXid writer = simc_begin(); + HostXid updater; + txn_vacuum_tests_entry_t entry; + + for (uint64 key = 1; key <= 14; key++) + { + uint64 tid = txn_vacuum_tests_inserted(io, store, held, database, key, 1036, writer); + + STD_ASSERT_EQ_U64(1165, 13, tid, ((key - 1) / 7) << 16 | ((key - 1) % 7 + 1)); + } + if (committed != 0) + txn_vacuum_tests_committed_status(io, writer, committed, __LINE__, 9); + txn_vacuum_tests_ended(io, writer, true); + updater = simc_begin(); + STD_ASSERT_EQ_U64(1169, 9, txn_vacuum_tests_updated(io, store, held, database, 1, 1, 1036, updater, NULL), + 2 << 16 | 1); + if (committed != 0) + txn_vacuum_tests_committed_status(io, updater, committed, __LINE__, 9); + txn_vacuum_tests_ended(io, updater, true); + entry = txn_vacuum_tests_entry(io, held, 0); + if (!entry.has) + STD_OPTION_UNWRAP_FAILED(1171, 36); + STD_ASSERT(1171, 9, entry.free.full, "entry(io, held, 0).unwrap().full"); + return updater; +} + +/* txn_vacuum_tests_versions_len returns how many versions of a key's row the delta holds, under a hold taken and given back at the lines the case unwraps them. */ +static size_t +txn_vacuum_tests_versions_len(HostIoIo *io, const HostSharedHeld *held, uint64 key, uint32 hold_line, + uint32 hold_col, uint32 line, uint32 col) +{ + HostDeltaHold hold; + HostError error; + StdVec chain; + uint8 bytes[8]; + size_t len; + + if (!host_delta_hold(io, &hold, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(hold_line, hold_col, &error); + std_u64_to_be_bytes(key, bytes); + if (!txn_memtable_table_versions(io, &hold, host_shared_Held_record(held), bytes, sizeof(bytes), &chain, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); + len = chain.len; + host_delta_Hold_drop(&hold); + txn_memtable_version_versions_drop(&chain, 0); + return len; +} + +/* txn_vacuum_tests_pruning_t is the take-time prune case's captures: block 0's entry after the full update. */ +typedef struct txn_vacuum_tests_pruning_t +{ + txn_vacuum_tests_entry_t moved; +} txn_vacuum_tests_pruning_t; + +/* txn_vacuum_tests_pruning_primary is the take-time prune case's primary: the next update of a row on the full block prunes it first and stays on it. */ +static void +txn_vacuum_tests_pruning_primary(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static std_TupleU32U16U16U64_t dead[] = {{0, 1, 1, 1036}}; + txn_vacuum_tests_pruning_t *c = captures; + HostXid updater = txn_vacuum_tests_filled(io, store, held, 9436, 0); + HostXid next; + SimcTaken horizon; + SimcTaken any; + HostSnapshot horizon_lent; + HostSnapshot any_lent; + TxnVacuumPrune prune; + txn_vacuum_tests_entry_t block; + StdVec records; + TxnRecordTidsFreed want; + const TxnRecordTidsFreed *const wanted[] = {&want}; + + c->moved = txn_vacuum_tests_entry(io, held, 0); + next = simc_begin(); + horizon = simc_non_vacuumable(next); + any = simc_snapshot(HOST_SNAPSHOT_KIND_ANY, false); + horizon_lent = simc_Taken_lent(&horizon); + any_lent = simc_Taken_lent(&any); + prune.horizon = &horizon_lent; + prune.any = &any_lent; + STD_ASSERT_EQ_U64(1191, 13, txn_vacuum_tests_updated(io, store, held, 9436, 3, 3, 1036, next, &prune), 8); + txn_vacuum_tests_ended(io, next, true); + block = txn_vacuum_tests_entry(io, held, 0); + if (!block.has) + STD_OPTION_UNWRAP_FAILED(1193, 44); + STD_ASSERT(1194, 13, !block.free.full, "!block.full"); + STD_ASSERT(1195, 13, !txn_free_space_Free_has(&block.free, 1), "!block.has(1)"); + STD_ASSERT_EQ_U64(1198, 13, txn_vacuum_tests_versions_len(io, held, 1, 1196, 46, 1197, 88), 1); + records = txn_vacuum_tests_logged(9436); + want = (TxnRecordTidsFreed) + { + .storage = txn_insert_tests_stored(9436), .horizon = updater.f0, .dead = + { + dead, 1, 1 + }, + .enders = + { + &updater.f0, 1, 1 + } + }; + txn_vacuum_tests_frees_is(1208, 13, &records, wanted, lengthof(wanted)); + txn_vacuum_tests_logged_free(&records); + simc_Taken_drop(&any); + simc_Taken_drop(&horizon); +} + +/* txn_vacuum_tests_pruning_replica is the take-time prune case's replica: redoing the primary's rows up to the full update sets the full flag too. */ +static void +txn_vacuum_tests_pruning_replica(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + txn_vacuum_tests_pruning_t *c = captures; + StdVec records = txn_vacuum_tests_logged(9436); + + for (size_t k = 0; k < records.len; k++) + { + txn_vacuum_tests_logged_t *record = &STD_VEC_AT(&records, txn_vacuum_tests_logged_t, k); + bool update; + + if (record->freed) + continue; + update = record->row.has_replaced; + record->row.storage = txn_insert_tests_stored(9437); + txn_vacuum_tests_redo(io, store, held, &record->row, record->end, 1218, 74); + if (update) + break; + } + txn_vacuum_tests_logged_free(&records); + txn_vacuum_tests_entry_is(1223, 13, txn_vacuum_tests_entry(io, held, 0), c->moved); +} + +/* + * Bugs row 75 (R1, R2): an update whose new row left block 0 sets the + * block's full flag, which a replica that redoes the primary's records sets + * too; the next update of a row on block 0, past every snapshot's horizon, + * prunes the block first, as heap_page_prune_opt does on a full page: key 1's + * old version leaves its chain, its 1036 bytes come back with its offset + * kept, logged with the updater's xid as the conflict horizon, the flag + * clears, and the new row takes (0,8), as heap's does. + */ +void +test_txn_vacuum_tests_a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block(void) +{ + SimcTestIo test_io = simc_io(); + txn_vacuum_tests_pruning_t c = {0}; + + txn_vacuum_tests_with(&test_io.io, 9436, txn_vacuum_tests_pruning_primary, &c); + txn_vacuum_tests_with(&test_io.io, 9437, txn_vacuum_tests_pruning_replica, &c); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_held_back is the held-back case: a snapshot that may see key 1's old version keeps it, and the next update leaves block 0. */ +static void +txn_vacuum_tests_held_back(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + HostXid updater = txn_vacuum_tests_filled(io, store, held, 9438, 0); + SimcTaken horizon = simc_non_vacuumable(updater); + SimcTaken any = simc_snapshot(HOST_SNAPSHOT_KIND_ANY, false); + HostSnapshot horizon_lent = simc_Taken_lent(&horizon); + HostSnapshot any_lent = simc_Taken_lent(&any); + TxnVacuumPrune prune = {&horizon_lent, &any_lent}; + HostXid next = simc_begin(); + + STD_ASSERT_EQ_U64(1239, 13, txn_vacuum_tests_updated(io, store, held, 9438, 3, 3, 1036, next, &prune), + 2 << 16 | 2); + txn_vacuum_tests_ended(io, next, true); + STD_ASSERT_EQ_U64(1242, 13, txn_vacuum_tests_versions_len(io, held, 1, 1241, 46, 1242, 87), 2); + simc_Taken_drop(&any); + simc_Taken_drop(&horizon); +} + +/* + * Bugs row 75's held-back case: a snapshot that may still see key 1's old + * version keeps it, as heap's prune keeps a tuple its horizon doesn't pass, + * so the next update of a row on block 0 finds no room there and its new row + * goes to (2,2). + */ +void +test_txn_vacuum_tests_a_take_leaves_a_version_an_open_snapshot_may_see(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9438, txn_vacuum_tests_held_back, NULL); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_claimed_is asserts at a line and column that the dead tuple ids a free claimed are those wanted, and frees them. */ +static void +txn_vacuum_tests_claimed_is(uint32 line, uint32 col, StdVec *claimed, const uint64 *want, size_t want_len) +{ + StdVec wanted = {(void *) want, want_len, want_len}; + bool eq = claimed->len == want_len && memcmp(claimed->ptr, want, want_len * sizeof(uint64)) == 0; + + STD_ASSERT_EQ(line, col, eq, claimed, chunks_trailer_VecU64_Debug_fmt, &wanted, chunks_trailer_VecU64_Debug_fmt); + std_vec_free_Checked(claimed, sizeof(uint64), _Alignof(uint64)); +} + +/* txn_vacuum_tests_claimed_once is the claimed once case: a second give-back of a dead tuple id claims nothing. */ +static void +txn_vacuum_tests_claimed_once(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed want[] = {{2, 300, 0}}; + static const uint64 two[] = {2}; + HostXid one = simc_begin(); + HostXid second; + StdVec found; + StdVec claimed; + + STD_ASSERT_EQ_U64(1254, 13, txn_vacuum_tests_inserted(io, store, held, 9439, 1, 100, one), 1); + txn_vacuum_tests_ended(io, one, true); + second = simc_begin(); + STD_ASSERT_EQ_U64(1257, 13, txn_vacuum_tests_inserted(io, store, held, 9439, 2, 300, second), 2); + txn_vacuum_tests_ended(io, second, false); + found = txn_vacuum_tests_found_every(io, store, held, 1259, 79); + txn_vacuum_tests_found_is(1260, 13, &found, want, lengthof(want)); + claimed = txn_vacuum_tests_claimed(io, store, held, 9439, found.ptr, found.len, true, 1261, 100); + txn_vacuum_tests_claimed_is(1261, 13, &claimed, two, lengthof(two)); + claimed = txn_vacuum_tests_claimed(io, store, held, 9439, found.ptr, found.len, true, 1262, 100); + txn_vacuum_tests_claimed_is(1262, 13, &claimed, NULL, 0); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_entry_is(1263, 13, txn_vacuum_tests_entry(io, held, 0), + txn_vacuum_tests_some(txn_free_space_Free_new(8168 - 400 + 300, NULL, 0, 291))); +} + +/* + * PRevPF 7's R1: a dead tuple id's marker is its one claim, so a second + * give-back of it, as a concurrent prune's, claims nothing and gives its 300 + * bytes back once. + */ +void +test_txn_vacuum_tests_a_dead_tuple_id_is_claimed_once(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9439, txn_vacuum_tests_claimed_once, NULL); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_refused_holds reports whether key 5's chain holds a row version at tuple id 5. */ +static bool +txn_vacuum_tests_refused_holds(HostIoIo *io, const HostSharedHeld *held) +{ + static const uint8 five[8] = {0, 0, 0, 0, 0, 0, 0, 5}; + HostDeltaHold hold; + HostError error; + StdVec chain; + bool holds = false; + + if (!host_delta_hold(io, &hold, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1274, 46, &error); + if (!txn_memtable_table_versions(io, &hold, host_shared_Held_record(held), five, sizeof(five), &chain, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1275, 88, &error); + for (size_t k = 0; !holds && k < chain.len; k++) + { + TxnVisibleRowVersion row; + + holds = txn_visible_RowVersion_of(&STD_VEC_AT(&chain, TxnMemtableVersionVersion, k), &row) && + row.put.tuple.f0 == 5; + } + txn_memtable_version_versions_drop(&chain, 0); + host_delta_Hold_drop(&hold); + return holds; +} + +/* txn_vacuum_tests_refused_primary is the refused marker case's primary: a dead tuple id on a full block keeps its row version. */ +static void +txn_vacuum_tests_refused_primary(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed dead[] = {{5, 28, 0}}; + HostXid writer = simc_begin(); + HostXid updater; + txn_vacuum_tests_entry_t none = {0}; + StdVec claimed; + + for (uint64 key = 1; key <= 292; key++) + txn_vacuum_tests_inserted(io, store, held, 9440, key, 28, writer); + txn_vacuum_tests_ended(io, writer, true); + updater = simc_begin(); + STD_ASSERT_EQ_U64(1285, 13, txn_vacuum_tests_updated(io, store, held, 9440, 5, 5, 28, updater, NULL), + 1 << 16 | 2); + txn_vacuum_tests_ended(io, updater, true); + txn_vacuum_tests_entry_is(1287, 13, txn_vacuum_tests_entry(io, held, 0), none); + claimed = txn_vacuum_tests_claimed(io, store, held, 9440, dead, lengthof(dead), true, 1289, 112); + txn_vacuum_tests_claimed_is(1288, 13, &claimed, NULL, 0); + STD_ASSERT(1292, 13, txn_vacuum_tests_refused_holds(io, held), "holds(io, held)"); +} + +/* txn_vacuum_tests_redo_all redoes every row change and free a database logged on another database's storage. */ +static void +txn_vacuum_tests_redo_all(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 from, uint32 to, + uint32 row_line, uint32 row_col, uint32 free_line, uint32 free_col) +{ + StdVec records = txn_vacuum_tests_logged(from); + + for (size_t k = 0; k < records.len; k++) + { + txn_vacuum_tests_logged_t *record = &STD_VEC_AT(&records, txn_vacuum_tests_logged_t, k); + + if (record->freed) + { + record->free.storage = txn_insert_tests_stored(to); + txn_vacuum_tests_redone(io, store, held, &record->free, record->end, free_line, free_col); + } + else + { + record->row.storage = txn_insert_tests_stored(to); + txn_vacuum_tests_redo(io, store, held, &record->row, record->end, row_line, row_col); + } + } + txn_vacuum_tests_logged_free(&records); +} + +/* txn_vacuum_tests_refused_replica is the refused marker case's replica: redoing the primary's records keeps the row version too. */ +static void +txn_vacuum_tests_refused_replica(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + txn_vacuum_tests_redo_all(io, store, held, 9440, 9441, 1299, 82, 1303, 81); + STD_ASSERT(1307, 13, txn_vacuum_tests_refused_holds(io, held), "holds(io, held)"); +} + +/* + * PRevPF 7's R2b: a dead tuple id whose marker refuses, here on a full block + * other than the fresh page, keeps its row version on the primary, and a + * storage that redoes the primary's records keeps it too. + */ +void +test_txn_vacuum_tests_a_refused_markers_redo_keeps_the_version_as_the_primary_does(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9440, txn_vacuum_tests_refused_primary, NULL); + txn_vacuum_tests_with(&test_io.io, 9441, txn_vacuum_tests_refused_replica, NULL); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_hint returns a storage's hint for a category, unwrapped at a line and column. */ +static uint32 +txn_vacuum_tests_hint(const HostSharedHeld *held, uint8 category, uint32 line, uint32 col) +{ + HostError error; + uint32 hint; + + if (!txn_state_hint(host_shared_Held_record(held), category, &hint, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); + return hint; +} + +/* txn_vacuum_tests_passed inserts five rows of 2700 heap bytes, the second aborted, so a search passes block 0 and moves its category's hint to 1. */ +static void +txn_vacuum_tests_passed(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database, + uint8 category) +{ + static const struct + { + uint64 key; + bool committed; + } rows[] = {{1, true}, {2, false}, {3, true}, {4, true}, {5, true}}; + + for (size_t k = 0; k < lengthof(rows); k++) + { + HostXid writer = simc_begin(); + + txn_vacuum_tests_inserted(io, store, held, database, rows[k].key, 2700, writer); + txn_vacuum_tests_ended(io, writer, rows[k].committed); + } + STD_ASSERT_EQ_U64(1327, 13, txn_vacuum_tests_hint(held, category, 1327, 61), 1); +} + +/* txn_vacuum_tests_raised is the raise case's free of a never-held tuple id, which lowers the hint back to block 0. */ +static void +txn_vacuum_tests_raised(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + static const TxnVacuumFreed want[] = {{2, 2700, 0}}; + uint8 category = *(const uint8 *) captures; + HostXid writer; + StdVec found; + + txn_vacuum_tests_passed(io, store, held, 9448, category); + found = txn_vacuum_tests_found_every(io, store, held, 1331, 79); + txn_vacuum_tests_found_is(1332, 13, &found, want, lengthof(want)); + txn_vacuum_tests_freed(io, store, held, 9448, &found, false, 1333, 90); + txn_vacuum_tests_found_free(&found); + STD_ASSERT_EQ_U64(1334, 13, txn_vacuum_tests_hint(held, category, 1334, 61), 0); + writer = simc_begin(); + STD_ASSERT_EQ_U64(1336, 13, txn_vacuum_tests_inserted(io, store, held, 9448, 6, 2700, writer), 2); + txn_vacuum_tests_ended(io, writer, true); +} + +/* txn_vacuum_tests_raised_dead is the raise case's give-back of a dead tuple id's bytes, which lowers the hint back to block 0. */ +static void +txn_vacuum_tests_raised_dead(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + uint8 category = *(const uint8 *) captures; + StdVec found; + + txn_vacuum_tests_passed(io, store, held, 9449, category); + found = txn_vacuum_tests_found_every(io, store, held, 1341, 79); + txn_vacuum_tests_freed(io, store, held, 9449, &found, true, 1342, 89); + txn_vacuum_tests_found_free(&found); + STD_ASSERT_EQ_U64(1343, 13, txn_vacuum_tests_hint(held, category, 1343, 61), 0); +} + +/* txn_vacuum_tests_raised_redo is the raise case's redo of a free on another storage, which lowers the hint back to block 0. */ +static void +txn_vacuum_tests_raised_redo(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + uint8 category = *(const uint8 *) captures; + StdVec records = txn_vacuum_tests_logged(9448); + + for (size_t k = 0; k < records.len; k++) + { + txn_vacuum_tests_logged_t *record = &STD_VEC_AT(&records, txn_vacuum_tests_logged_t, k); + HostError error; + bool advanced; + + if (!record->freed) + { + record->row.storage = txn_insert_tests_stored(9450); + txn_vacuum_tests_redo(io, store, held, &record->row, record->end, 1350, 82); + continue; + } + record->free.storage = txn_insert_tests_stored(9450); + if (!txn_state_advanced(host_shared_Held_record(held), category, 0, 1, &advanced, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1354, 82, &error); + STD_ASSERT(1354, 25, advanced, "state::advanced(held.record(), category, (0, 1)).unwrap()"); + txn_vacuum_tests_redone(io, store, held, &record->free, record->end, 1355, 81); + STD_ASSERT_EQ_U64(1356, 25, txn_vacuum_tests_hint(held, category, 1356, 73), 0); + break; + } + txn_vacuum_tests_logged_free(&records); +} + +/* + * Bugs row 83's raise paths: a search for room passes a full block that + * isn't the fresh page and moves its category's hint past it; every free + * that raises the block's entry then lowers the hint back to it, a + * never-held tuple id's free, a dead tuple id's give-back of its bytes, and + * the redo of a free on another storage, so the next search finds the block. + */ +void +test_txn_vacuum_tests_every_raise_of_a_blocks_room_lowers_its_categorys_hint(void) +{ + SimcTestIo test_io = simc_io(); + HostError error; + uint8 category; + + if (!txn_free_space_needed(2700, &category, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1320, 49, &error); + txn_vacuum_tests_with(&test_io.io, 9448, txn_vacuum_tests_raised, &category); + txn_vacuum_tests_with(&test_io.io, 9449, txn_vacuum_tests_raised_dead, &category); + txn_vacuum_tests_with(&test_io.io, 9450, txn_vacuum_tests_raised_redo, &category); + simc_TestIo_drop(&test_io); +} + +/* + * txn_vacuum_tests_committed inserts a key's row of 100 heap bytes into a + * database's storage as a transaction whose commit record ends at 100, and + * returns its tuple id. + */ +static uint64 +txn_vacuum_tests_committed(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database, + uint64 key) +{ + HostXid xid = simc_begin(); + uint64 tid = txn_vacuum_tests_inserted(io, store, held, database, key, 100, xid); + + txn_vacuum_tests_committed_status(io, xid, 100, 1369, 78); + txn_vacuum_tests_ended(io, xid, true); + return tid; +} + +/* txn_vacuum_tests_made_t is a case's keyless table's storage: its repository, a reader of the shared state, and its state. */ +typedef struct txn_vacuum_tests_made_t +{ + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; +} txn_vacuum_tests_made_t; + +/* + * txn_vacuum_tests_made makes a keyless table's storage in a new database, + * its repository, reader, and state unwrapped at the lines and columns the + * case gives. + */ +static void +txn_vacuum_tests_made(HostIoIo *io, uint32 database, uint32 create_line, uint32 create_col, uint32 read_line, + uint32 read_col, uint32 state_line, uint32 state_col, txn_vacuum_tests_made_t *made) +{ + HostStorage storage = txn_insert_tests_stored(database); + TxnStateMade table = {.key = {.tag = TXN_BASE_KEY_NONE}, .table = 16384}; + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_create(io, database, HOST_PG_DEFAULT, &made->repo, &failed)) + STD_RESULT_EXPECT_FAILED(create_line, create_col, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!host_shared_read(io, &made->reading, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(read_line, read_col, &error); + if (!txn_state_state(io, &made->repo, &made->reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, true, &table, + &made->held, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(state_line, state_col, &error); +} + +/* txn_vacuum_tests_made_drop drops a case's storage's state, reader, and repository. */ +static void +txn_vacuum_tests_made_drop(txn_vacuum_tests_made_t *made) +{ + host_shared_Held_drop(&made->held); + host_shared_Reader_drop(&made->reading); + repo_Repo_drop(&made->repo); +} + +/* txn_vacuum_tests_folded reports whether a fold of a keyless storage up to 200 installed a base, unwrapped at a line and column. */ +static bool +txn_vacuum_tests_folded(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, + const HostSharedHeld *held, uint32 line, uint32 col) +{ + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + HostError error; + bool folded; + + if (!txn_fold_folded(io, repo, reading, held, &order, false, 200, &TXN_VACUUM_TESTS_PAGE, NULL, NULL, &folded, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, col, &error); + return folded; +} + +/* + * The fold's drop beside VACUUM (bugs row 78): a row a fold applied and + * dropped from the delta keeps its tuple id held, since the installed base's + * tuple id tree pairs it, so VACUUM frees nothing. + */ +void +test_txn_vacuum_tests_a_folded_rows_tuple_id_is_not_freed(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_made_t made; + TreesStoreReader store; + StdVec found; + + txn_vacuum_tests_made(io, 9446, 1381, 87, 1382, 46, 1383, 106, &made); + store = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + STD_ASSERT_EQ_U64(1385, 9, + txn_vacuum_tests_committed(io, TREES_STORE_READER_NODE_STORE(&store), &made.held, 9446, 1), 1); + STD_ASSERT(1387, 9, txn_vacuum_tests_folded(io, &made.repo, &made.reading, &made.held, 1387, 101), + "crate::fold::folded(io, (&repo, &reading), &held, &order, false, 200, (None, None)).unwrap()"); + found = txn_vacuum_tests_found_every(io, TREES_STORE_READER_NODE_STORE(&store), &made.held, 1388, 76); + txn_vacuum_tests_found_is(1388, 9, &found, NULL, 0); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_opened opens a database's repository and a reader of the shared state, as an actor of a fold race reads them. */ +static bool +txn_vacuum_tests_opened(HostIoIo *io, uint32 database, RepoRepo *repo, HostSharedReader *reading, HostError *error) +{ + ChunksErrorError failed; + + if (!repo_Repo_open(io, database, HOST_PG_DEFAULT, repo, &failed)) + { + *error = txn_base_storage(&failed); + return false; + } + if (!host_shared_read(io, reading, error)) + { + repo_Repo_drop(repo); + return false; + } + return true; +} + +/* txn_vacuum_tests_racing_state opens the fold race's storage's state, dropping its repository and reader on a failure. */ +static bool +txn_vacuum_tests_racing_state(HostIoIo *io, uint32 database, RepoRepo *repo, HostSharedReader *reading, + HostSharedHeld *held, HostError *error) +{ + HostStorage storage = txn_insert_tests_stored(database); + + if (!txn_vacuum_tests_opened(io, database, repo, reading, error)) + return false; + if (!txn_state_state(io, repo, reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, held, error)) + { + host_shared_Reader_drop(reading); + repo_Repo_drop(repo); + return false; + } + return true; +} + +/* txn_vacuum_tests_racing_vacuum is the fold race's VACUUM, of the database its captures name: its find of never-held tuple ids finds none. */ +static bool +txn_vacuum_tests_racing_vacuum(void *captures, HostIoIo *io, HostError *error) +{ + static const char text[] = "VACUUM freed a folded row's tuple id"; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + SimcTaken every; + HostSnapshot lent; + StdVec found; + bool ok; + + if (!txn_vacuum_tests_racing_state(io, *(const uint32 *) captures, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + every = txn_vacuum_tests_every(); + lent = simc_Taken_lent(&every); + ok = txn_vacuum_never_held(io, TREES_STORE_READER_NODE_STORE(&store), &held, &TXN_VACUUM_TESTS_PAGE, &lent, + &found, error); + simc_Taken_drop(&every); + } + if (ok) + { + if (found.len != 0) + { + StdString message; + + std_string_from_Checked(&message, text, sizeof(text) - 1); + *error = host_Error_fork((const uint8 *) "XX000", &message); + ok = false; + } + txn_vacuum_tests_found_free(&found); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* txn_vacuum_tests_racing_folder is a fold race's folder, of the database its captures name: it folds the storage's committed rows. */ +static bool +txn_vacuum_tests_racing_folder(void *captures, HostIoIo *io, HostError *error) +{ + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + bool folded; + bool ok; + + if (!txn_vacuum_tests_racing_state(io, *(const uint32 *) captures, &repo, &reading, &held, error)) + return false; + ok = txn_fold_folded(io, &repo, &reading, &held, &order, false, 200, &TXN_VACUUM_TESTS_PAGE, NULL, NULL, &folded, error); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* txn_vacuum_tests_endings_Debug_fmt is a Vec<(String, Ending)>'s Debug. */ +static bool +txn_vacuum_tests_endings_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *endings = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < endings->len; k++) + { + const simc_TupleStringEnding_t *ending = &STD_VEC_AT(endings, simc_TupleStringEnding_t, k); + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&ending->f0, std_String_Debug_fmt), + STD_ITEM(&ending->f1, simc_Ending_Debug_fmt)); + + std_DebugList_entry(&list, &tuple, std_tuple_Debug_fmt); + } + return std_DebugList_finish(&list); +} + +/* txn_vacuum_tests_all_returned reports whether a run ended with every actor returning Ok. */ +bool +txn_vacuum_tests_all_returned(const SimcReport *report) +{ + bool returned = report->outcome.tag == SIMC_OUTCOME_ENDED; + + for (size_t k = 0; returned && k < report->outcome.u.ended.len; k++) + { + const SimcEnding *ending = &STD_VEC_AT(&report->outcome.u.ended, simc_TupleStringEnding_t, k).f1; + + returned = ending->tag == SIMC_ENDING_RETURNED && !ending->is_err; + } + return returned; +} + +/* + * txn_vacuum_tests_returned asserts that a run ended with every actor + * returning Ok, panicking at a test's line and column for its outcome and at + * another for its endings. + */ +void +txn_vacuum_tests_returned(const SimcReport *report, const char *file, uint32 ended_line, uint32 ended_col, + uint32 line, uint32 col) +{ + if (report->outcome.tag != SIMC_OUTCOME_ENDED) + { + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&report->outcome, simc_Outcome_Debug_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_panic_fmt(file, ended_line, ended_col, &message); + } + if (!txn_vacuum_tests_all_returned(report)) + { + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&report->outcome.u.ended, txn_vacuum_tests_endings_Debug_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + + std_panic_fmt(file, line, col, &message); + } +} + +/* + * PRevPF 7's generation re-read: a fold that installs and drops between + * VACUUM's read of a block's takes and its mapped checks moves the base + * generation, so the find reads the block again under the newest base and + * frees nothing of the folded row. + */ +void +test_txn_vacuum_tests_a_fold_between_the_finds_reads_leaves_its_rows_held(void) +{ + static const SimcStep steps[] = {{"folder", 6, UINT64_MAX}, {"vacuum", 6, UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 database = 9447; + SimcActor vacuum = {txn_vacuum_tests_racing_vacuum, &database}; + SimcActor folder = {txn_vacuum_tests_racing_folder, &database}; + SimcRun run; + SimcReport report; + + { + txn_vacuum_tests_made_t made; + TreesStoreReader store; + + txn_vacuum_tests_made(io, 9447, 1400, 91, 1401, 50, 1402, 110, &made); + store = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + STD_ASSERT_EQ_U64(1404, 13, + txn_vacuum_tests_committed(io, TREES_STORE_READER_NODE_STORE(&store), &made.held, 9447, 1), + 1); + txn_vacuum_tests_made_drop(&made); + } + run = simc_Run_new(81); + simc_Run_actor(&run, "vacuum", 6, vacuum); + simc_Run_actor(&run, "folder", 6, folder); + simc_Run_until(&run, "vacuum", 6, "doltgres-vacuum-found"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, RUST_FILE, 1430, 67, 1431, 9); + simc_Report_drop(&report); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_prune_updated updates key 3's row of a database's full block 0 as a new transaction's, with a take-time prune, and returns its new row's tuple id. */ +static uint64 +txn_vacuum_tests_prune_updated(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database) +{ + HostXid next = simc_begin(); + SimcTaken horizon = simc_non_vacuumable(next); + SimcTaken any = simc_snapshot(HOST_SNAPSHOT_KIND_ANY, false); + HostSnapshot horizon_lent = simc_Taken_lent(&horizon); + HostSnapshot any_lent = simc_Taken_lent(&any); + TxnVacuumPrune prune = {&horizon_lent, &any_lent}; + uint64 tid = txn_vacuum_tests_updated(io, store, held, database, 3, 3, 1036, next, &prune); + + txn_vacuum_tests_ended(io, next, true); + simc_Taken_drop(&any); + simc_Taken_drop(&horizon); + return tid; +} + +/* txn_vacuum_tests_raced_t is the prune race's captures: its database, then the update's new row's tuple id. */ +typedef struct txn_vacuum_tests_raced_t +{ + uint32 database; + uint64 tid; +} txn_vacuum_tests_raced_t; + +/* txn_vacuum_tests_racing_updater is the prune race's update of key 3, whose new row's tuple id it writes into its captures. */ +static bool +txn_vacuum_tests_racing_updater(void *captures, HostIoIo *io, HostError *error) +{ + txn_vacuum_tests_raced_t *raced = captures; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + + if (!txn_vacuum_tests_racing_state(io, raced->database, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + raced->tid = txn_vacuum_tests_prune_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, + raced->database); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return true; +} + +/* txn_vacuum_tests_committed_at ends a transaction committed, its status entry's commit record ending at an LSN. */ +static void +txn_vacuum_tests_committed_at(HostIoIo *io, HostXid xid, HostLsn end) +{ + txn_vacuum_tests_committed_status(io, xid, end, __LINE__, 5); + txn_vacuum_tests_ended(io, xid, true); +} + +/* txn_vacuum_tests_folded_at folds a case's keyless storage up to a point, asserting that it installed a base. */ +static void +txn_vacuum_tests_folded_at(HostIoIo *io, const txn_vacuum_tests_made_t *made, HostLsn point) +{ + TreesKeyOrder order = {.tag = TREES_KEY_ORDER_BYTES}; + HostError error; + bool installed; + + if (!txn_fold_folded(io, &made->repo, &made->reading, &made->held, &order, false, point, &TXN_VACUUM_TESTS_PAGE, + NULL, NULL, &installed, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + STD_ASSERT(__LINE__, 5, installed, "installed"); +} + +/* + * txn_vacuum_tests_filled_at makes a new storage of a database whose base at + * 150 holds block 0 full: fourteen rows of 1036 bytes committed at 100 and + * folded at 120, then key 1's update to (2,1), committed at 130, which flags + * the block. Key 2's update to 100 bytes, committed at 175, then takes (0,8) + * in the delta; when folded, a fold at 200 takes that take out of the delta. + */ +static void +txn_vacuum_tests_filled_at(HostIoIo *io, uint32 database, bool folded) +{ + txn_vacuum_tests_made_t made; + TreesStoreNodeStore store; + TreesStoreReader reader; + HostXid writer; + HostXid updater; + HostXid shrinker; + + txn_vacuum_tests_made(io, database, __LINE__, 2, __LINE__, 2, __LINE__, 2, &made); + reader = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + store = TREES_STORE_READER_NODE_STORE(&reader); + writer = simc_begin(); + for (uint64 key = 1; key <= 14; key++) + (void) txn_vacuum_tests_inserted(io, store, &made.held, database, key, 1036, writer); + txn_vacuum_tests_committed_at(io, writer, 100); + txn_vacuum_tests_folded_at(io, &made, 120); + updater = simc_begin(); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_updated(io, store, &made.held, database, 1, 1, 1036, updater, NULL), + 2 << 16 | 1); + txn_vacuum_tests_committed_at(io, updater, 130); + txn_vacuum_tests_folded_at(io, &made, 150); + shrinker = simc_begin(); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_updated(io, store, &made.held, database, 2, 2, 100, shrinker, NULL), + 8); + txn_vacuum_tests_committed_at(io, shrinker, 175); + if (folded) + txn_vacuum_tests_folded_at(io, &made, 200); + txn_vacuum_tests_made_drop(&made); +} + +/* txn_vacuum_tests_block_0 returns a database's storage's block 0 entry, read through a state of its own. */ +static txn_vacuum_tests_entry_t +txn_vacuum_tests_block_0(HostIoIo *io, uint32 database) +{ + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostError error; + txn_vacuum_tests_entry_t entry; + + if (!txn_vacuum_tests_racing_state(io, database, &repo, &reading, &held, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + entry = txn_vacuum_tests_entry(io, &held, 0); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return entry; +} + +/* + * Bugs row 240's prune: a fold that installs a base with block 0's full flag + * and drops the flag's version between a take-time prune's read of the older + * base and its read of the delta moves the base generation, so the prune + * reads the block again under the newest base and clears the flag, and the + * update's new row and block 0's entry are a twin storage's that folds + * before the same update. + */ +void +test_txn_vacuum_tests_a_fold_between_a_prunes_reads_leaves_what_a_prune_after_it_leaves(void) +{ + static const SimcStep steps[] = {{"folder", 6, UINT64_MAX}, {"updater", 7, UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_raced_t raced = {.database = 9463}; + SimcActor updater = {txn_vacuum_tests_racing_updater, &raced}; + SimcActor folder = {txn_vacuum_tests_racing_folder, &raced.database}; + uint64 twin; + SimcRun run; + SimcReport report; + + txn_vacuum_tests_filled_at(io, 9466, true); + { + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + HostError error; + + if (!txn_vacuum_tests_racing_state(io, 9466, &repo, &reading, &held, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + twin = txn_vacuum_tests_prune_updated(io, TREES_STORE_READER_NODE_STORE(&store), &held, 9466); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + } + txn_vacuum_tests_filled_at(io, 9463, false); + run = simc_Run_new(83); + simc_Run_actor(&run, "updater", 7, updater); + simc_Run_actor(&run, "folder", 6, folder); + simc_Run_until(&run, "updater", 7, "doltgres-prune-based"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, __FILE__, __LINE__, 2, __LINE__, 2); + simc_Report_drop(&report); + STD_ASSERT_EQ_U64(__LINE__, 5, raced.tid, twin); + txn_vacuum_tests_entry_is(__LINE__, 5, txn_vacuum_tests_block_0(io, 9463), txn_vacuum_tests_block_0(io, 9466)); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_counted_t is the spent count race's captures: its database, then block 0's count of spent places. */ +typedef struct txn_vacuum_tests_counted_t +{ + uint32 database; + uint64 count; +} txn_vacuum_tests_counted_t; + +/* txn_vacuum_tests_racing_counter is the spent count race's count of block 0's spent places, which it writes into its captures. */ +static bool +txn_vacuum_tests_racing_counter(void *captures, HostIoIo *io, HostError *error) +{ + txn_vacuum_tests_counted_t *counted = captures; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + bool ok; + + if (!txn_vacuum_tests_racing_state(io, counted->database, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + ok = txn_vacuum_spent(io, TREES_STORE_READER_NODE_STORE(&store), &held, 0, &TXN_VACUUM_TESTS_PAGE, + &counted->count, error); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* txn_vacuum_tests_committed_filled fills a new storage of a database as filled does, its commits at 100, folding it at 200 when folded. */ +static void +txn_vacuum_tests_committed_filled(HostIoIo *io, uint32 database, bool folded) +{ + txn_vacuum_tests_made_t made; + TreesStoreReader store; + + txn_vacuum_tests_made(io, database, __LINE__, 2, __LINE__, 2, __LINE__, 2, &made); + store = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + (void) txn_vacuum_tests_filled(io, TREES_STORE_READER_NODE_STORE(&store), &made.held, database, 100); + if (folded) + txn_vacuum_tests_folded_at(io, &made, 200); + txn_vacuum_tests_made_drop(&made); +} + +/* + * Bugs row 240's spent count: a fold that installs a base mapping block 0's + * live rows and drops their versions between ANALYZE's read of the base and + * its mapped checks, while key 1's dead row keeps the block's takes in the + * delta, moves the base generation, so the count reads the block again under + * the newest base and counts what a twin storage that folds first counts. + */ +void +test_txn_vacuum_tests_a_fold_during_a_blocks_spent_count_counts_what_a_count_after_it_counts(void) +{ + static const SimcStep steps[] = {{"folder", 6, UINT64_MAX}, {"counter", 7, UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_counted_t counted = {.database = 9467}; + txn_vacuum_tests_counted_t twin = {.database = 9451}; + SimcActor counter = {txn_vacuum_tests_racing_counter, &counted}; + SimcActor folder = {txn_vacuum_tests_racing_folder, &counted.database}; + HostError error; + SimcRun run; + SimcReport report; + + txn_vacuum_tests_committed_filled(io, 9451, true); + if (!txn_vacuum_tests_racing_counter(&twin, io, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + txn_vacuum_tests_committed_filled(io, 9467, false); + run = simc_Run_new(84); + simc_Run_actor(&run, "counter", 7, counter); + simc_Run_actor(&run, "folder", 6, folder); + simc_Run_until(&run, "counter", 7, "doltgres-vacuum-found"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, __FILE__, __LINE__, 2, __LINE__, 2); + simc_Report_drop(&report); + STD_ASSERT_EQ_U64(__LINE__, 5, counted.count, twin.count); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_finding_t is the find race's captures: its database, then the tuple ids VACUUM's find found. */ +typedef struct txn_vacuum_tests_finding_t +{ + uint32 database; + StdVec tids; +} txn_vacuum_tests_finding_t; + +/* txn_vacuum_tests_racing_finder is the find race's VACUUM, whose find's tuple ids it writes into its captures. */ +static bool +txn_vacuum_tests_racing_finder(void *captures, HostIoIo *io, HostError *error) +{ + txn_vacuum_tests_finding_t *finding = captures; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + SimcTaken every; + HostSnapshot lent; + StdVec found; + bool ok; + + if (!txn_vacuum_tests_racing_state(io, finding->database, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + every = txn_vacuum_tests_every(); + lent = simc_Taken_lent(&every); + ok = txn_vacuum_never_held(io, TREES_STORE_READER_NODE_STORE(&store), &held, &TXN_VACUUM_TESTS_PAGE, &lent, + &found, error); + simc_Taken_drop(&every); + } + if (ok) + { + std_vec_new(&finding->tids, sizeof(uint64), _Alignof(uint64)); + for (size_t i = 0; i < found.len; i++) + std_vec_push_Checked(&finding->tids, &STD_VEC_AT(&found, TxnVacuumFreed, i).f0, sizeof(uint64), + _Alignof(uint64)); + txn_vacuum_tests_found_free(&found); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * Bugs row 240's find: a fold that installs a base mapping block 0's live + * rows and drops their versions between VACUUM's read of the block's takes + * and its mapped checks, while key 1's dead row keeps the takes in the delta, + * moves the base generation, so the find reads the block again under the + * newest base and finds what a twin storage that folds first finds. + */ +void +test_txn_vacuum_tests_a_fold_during_a_find_finds_what_a_find_after_it_finds(void) +{ + static const SimcStep steps[] = {{"folder", 6, UINT64_MAX}, {"finder", 6, UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_finding_t finding = {.database = 9454}; + txn_vacuum_tests_finding_t twin = {.database = 9455}; + SimcActor finder = {txn_vacuum_tests_racing_finder, &finding}; + SimcActor folder = {txn_vacuum_tests_racing_folder, &finding.database}; + HostError error; + SimcRun run; + SimcReport report; + + txn_vacuum_tests_committed_filled(io, twin.database, true); + if (!txn_vacuum_tests_racing_finder(&twin, io, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + txn_vacuum_tests_committed_filled(io, finding.database, false); + run = simc_Run_new(86); + simc_Run_actor(&run, "finder", 6, finder); + simc_Run_actor(&run, "folder", 6, folder); + simc_Run_until(&run, "finder", 6, "doltgres-vacuum-found"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, __FILE__, __LINE__, 2, __LINE__, 2); + simc_Report_drop(&report); + STD_ASSERT_EQ_U64(__LINE__, 5, finding.tids.len, twin.tids.len); + for (size_t i = 0; i < twin.tids.len; i++) + STD_ASSERT_EQ_U64(__LINE__, 9, STD_VEC_AT(&finding.tids, uint64, i), STD_VEC_AT(&twin.tids, uint64, i)); + std_vec_free_Checked(&finding.tids, sizeof(uint64), _Alignof(uint64)); + std_vec_free_Checked(&twin.tids, sizeof(uint64), _Alignof(uint64)); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_racing_freer is the free race's VACUUM, of the database its captures name: it frees key 1's old tuple id, (0,1). */ +static bool +txn_vacuum_tests_racing_freer(void *captures, HostIoIo *io, HostError *error) +{ + uint32 database = *(const uint32 *) captures; + TxnVacuumFreed freed = {.f0 = 1, .f1 = 1036}; + HostStorage toast = {0}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + StdVec claimed; + bool ok; + + if (!txn_vacuum_tests_racing_state(io, database, &repo, &reading, &held, error)) + return false; + std_vec_new(&freed.f3, TXN_VACUUM_ID); + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + ok = txn_vacuum_freed(io, TREES_STORE_READER_NODE_STORE(&store), &held, NULL, toast, + txn_insert_tests_stored(database), true, &TXN_VACUUM_TESTS_PAGE, &freed, 1, false, &claimed, + error); + } + if (ok) + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + std_vec_free_Checked(&freed.f3, TXN_VACUUM_ID); + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * Bugs row 240's free: a fold that installs a base and drops a take of block + * 0's between VACUUM's read of the base and its give back of (0,1) moves the + * base generation, so the free reads the block's entry again under the + * newest base, and block 0's entry is a twin storage's that folds before the + * same free. + */ +void +test_txn_vacuum_tests_a_fold_during_a_free_leaves_what_a_free_after_it_leaves(void) +{ + static const SimcStep steps[] = {{"folder", 6, UINT64_MAX}, {"freer", 5, UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 database = 9452; + uint32 twin = 9453; + SimcActor freer = {txn_vacuum_tests_racing_freer, &database}; + SimcActor folder = {txn_vacuum_tests_racing_folder, &database}; + HostError error; + SimcRun run; + SimcReport report; + + txn_vacuum_tests_filled_at(io, twin, true); + if (!txn_vacuum_tests_racing_freer(&twin, io, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + txn_vacuum_tests_filled_at(io, database, false); + run = simc_Run_new(85); + simc_Run_actor(&run, "freer", 5, freer); + simc_Run_actor(&run, "folder", 6, folder); + simc_Run_until(&run, "freer", 5, "doltgres-vacuum-freeing"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, __FILE__, __LINE__, 2, __LINE__, 2); + simc_Report_drop(&report); + txn_vacuum_tests_entry_is(__LINE__, 5, txn_vacuum_tests_block_0(io, database), txn_vacuum_tests_block_0(io, twin)); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_redoing_t is the redo race's captures: its database, and the free's record it replays with its end. */ +typedef struct txn_vacuum_tests_redoing_t +{ + uint32 database; + const TxnRecordTidsFreed *free; + HostLsn end; +} txn_vacuum_tests_redoing_t; + +/* txn_vacuum_tests_racing_replayer is the redo race's replay of a free on the database its captures name. */ +static bool +txn_vacuum_tests_racing_replayer(void *captures, HostIoIo *io, HostError *error) +{ + const txn_vacuum_tests_redoing_t *redoing = captures; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + bool ok; + + if (!txn_vacuum_tests_racing_state(io, redoing->database, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + ok = txn_vacuum_redo_freed(io, TREES_STORE_READER_NODE_STORE(&store), &held, NULL, redoing->free, + &TXN_VACUUM_TESTS_PAGE, redoing->end, error); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * Bugs row 319's case: a free logs exactly what it gives back, (0,1)'s 1036 + * bytes, and its record, replayed with a fold that installs a base and drops + * a take of block 0's between the redo's read of the base and its give back, + * leaves block 0's entry that of the twin storage whose free logged it. Its + * databases are used by no other test, since the simulation's log holds every + * test's records. + */ +void +test_txn_vacuum_tests_a_fold_during_a_frees_redo_leaves_what_the_free_left(void) +{ + static const SimcStep steps[] = {{"folder", 6, UINT64_MAX}, {"redoer", 6, UINT64_MAX}}; + static std_TupleU32U16U16U64_t runs[] = {{0, 1, 1, 1036}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 twin = 9532; + txn_vacuum_tests_redoing_t redoing = {.database = 9531}; + static uint64 enders[] = {0}; + TxnRecordTidsFreed want = {.storage = txn_insert_tests_stored(twin), .runs = {runs, 1, 1}, .enders = {enders, 1, 1}}; + const TxnRecordTidsFreed *wants[] = {&want}; + SimcActor redoer = {txn_vacuum_tests_racing_replayer, &redoing}; + SimcActor folder = {txn_vacuum_tests_racing_folder, &redoing.database}; + HostError error; + StdVec records; + SimcRun run; + SimcReport report; + + txn_vacuum_tests_filled_at(io, twin, true); + if (!txn_vacuum_tests_racing_freer(&twin, io, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + records = txn_vacuum_tests_logged(twin); + for (size_t k = 0; k < records.len && redoing.free == NULL; k++) + { + const txn_vacuum_tests_logged_t *record = &STD_VEC_AT(&records, txn_vacuum_tests_logged_t, k); + + if (record->freed) + { + redoing.free = &record->free; + redoing.end = record->end; + } + } + txn_vacuum_tests_frees_is(__LINE__, 5, &records, wants, lengthof(wants)); + txn_vacuum_tests_filled_at(io, redoing.database, false); + run = simc_Run_new(87); + simc_Run_actor(&run, "redoer", 6, redoer); + simc_Run_actor(&run, "folder", 6, folder); + simc_Run_until(&run, "redoer", 6, "doltgres-vacuum-freeing"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, __FILE__, __LINE__, 2, __LINE__, 2); + simc_Report_drop(&report); + txn_vacuum_tests_entry_is(__LINE__, 5, txn_vacuum_tests_block_0(io, redoing.database), + txn_vacuum_tests_block_0(io, twin)); + txn_vacuum_tests_logged_free(&records); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_racing_redoer is the redo race's replay, at 300, of a free giving back (0,1)'s 1036 bytes on the database its captures name. */ +static bool +txn_vacuum_tests_racing_redoer(void *captures, HostIoIo *io, HostError *error) +{ + uint32 database = *(const uint32 *) captures; + std_TupleU32U16U16U64_t run = {0, 1, 1, 1036}; + TxnRecordTidsFreed free = {.storage = txn_insert_tests_stored(database), .runs = txn_vacuum_runs_view(&run, 1), + .dead = txn_vacuum_runs_view(NULL, 0)}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + bool ok; + + if (!txn_vacuum_tests_racing_state(io, database, &repo, &reading, &held, error)) + return false; + { + TreesStoreReader store = repo_Repo_reader(&repo, HOST_CACHE_PLACEMENT_SERVER, &reading); + + ok = txn_vacuum_redo_freed(io, TREES_STORE_READER_NODE_STORE(&store), &held, NULL, &free, + &TXN_VACUUM_TESTS_PAGE, 300, error); + } + host_shared_Held_drop(&held); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return ok; +} + +/* + * Bugs row 240's redo: a fold that installs a base and drops a take of block + * 0's between a free's redo's read of the base and its give back of (0,1) + * moves the base generation, so the redo reads the block's entry again under + * the newest base, and block 0's entry is a twin storage's that folds before + * the same redo. + */ +void +test_txn_vacuum_tests_a_fold_during_a_frees_redo_leaves_what_a_redo_after_it_leaves(void) +{ + static const SimcStep steps[] = {{"folder", 6, UINT64_MAX}, {"redoer", 6, UINT64_MAX}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint32 database = 9456; + uint32 twin = 9468; + SimcActor redoer = {txn_vacuum_tests_racing_redoer, &database}; + SimcActor folder = {txn_vacuum_tests_racing_folder, &database}; + HostError error; + SimcRun run; + SimcReport report; + + txn_vacuum_tests_filled_at(io, twin, true); + if (!txn_vacuum_tests_racing_redoer(&twin, io, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + txn_vacuum_tests_filled_at(io, database, false); + run = simc_Run_new(87); + simc_Run_actor(&run, "redoer", 6, redoer); + simc_Run_actor(&run, "folder", 6, folder); + simc_Run_until(&run, "redoer", 6, "doltgres-vacuum-freeing"); + simc_Run_script(&run, steps, lengthof(steps)); + report = simc_Run_finish(run); + txn_vacuum_tests_returned(&report, __FILE__, __LINE__, 2, __LINE__, 2); + simc_Report_drop(&report); + txn_vacuum_tests_entry_is(__LINE__, 5, txn_vacuum_tests_block_0(io, database), txn_vacuum_tests_block_0(io, twin)); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_deleted deletes a key's row of a storage as a transaction's, a delete version pushed onto the key's chain. */ +static void +txn_vacuum_tests_deleted(HostIoIo *io, const HostSharedHeld *held, uint64 key, HostXid xid) +{ + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_DELETE}; + TxnMemtableVersionChange change = {.writer = {.xid = xid.f0}, .op = &op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, + .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + HostError error; + uint8 bytes[8]; + + std_u64_to_be_bytes(key, bytes); + if (!txn_memtable_table_push(io, host_shared_Held_record(held), bytes, sizeof(bytes), &change, &handle, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1441, 69, &error); +} + +/* txn_vacuum_tests_horizons returns the conflict horizons of the frees a database logged, in their order. */ +static StdVec +txn_vacuum_tests_horizons(uint32 database) +{ + StdVec records = txn_vacuum_tests_logged(database); + StdVec horizons; + + std_vec_new(&horizons, sizeof(uint64), _Alignof(uint64)); + for (size_t k = 0; k < records.len; k++) + { + const txn_vacuum_tests_logged_t *record = &STD_VEC_AT(&records, txn_vacuum_tests_logged_t, k); + + if (record->freed) + std_vec_push_Checked(&horizons, &record->free.horizon, sizeof(uint64), _Alignof(uint64)); + } + txn_vacuum_tests_logged_free(&records); + return horizons; +} + +/* txn_vacuum_tests_u64s_is_with asserts at a line and column, with a message, that a Vec is the u64s wanted. */ +static void +txn_vacuum_tests_u64s_is_with(uint32 line, uint32 col, const StdVec *got, const uint64 *want, size_t want_len, + const StdFmtArguments *message) +{ + StdVec wanted = {(void *) want, want_len, want_len}; + + if (got->len != want_len || memcmp(got->ptr, want, want_len * sizeof(uint64)) != 0) + std_assert_failed(RUST_FILE, line, col, "==", got, chunks_trailer_VecU64_Debug_fmt, &wanted, + chunks_trailer_VecU64_Debug_fmt, message); +} + +/* txn_vacuum_tests_enders runs the dropped rows' case on a database: same is whether the rows' own writer ended them. */ +static void +txn_vacuum_tests_enders(HostIoIo *io, uint32 database, bool same) +{ + static const StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&same, std_bool_Display_fmt}}; + StdFmtArguments message = {pieces, lengthof(pieces), args}; + txn_vacuum_tests_made_t made; + TreesStoreReader reader; + TreesStoreNodeStore store; + HostXid writer; + uint64 tids[2]; + uint64 kept; + uint64 enders[2]; + uint64 kept_enders[2]; + StdVec takers; + StdVec stored_enders; + StdVec found; + StdVec horizons; + TxnVacuumFreed want[2]; + TxnBaseBase base; + HostError error; + bool any = false; + bool some; + + txn_vacuum_tests_made(io, database, 1456, 95, 1457, 50, 1459, 107, &made); + reader = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + store = TREES_STORE_READER_NODE_STORE(&reader); + writer = simc_begin(); + for (uint64 key = 1; key <= 2; key++) + tids[key - 1] = txn_vacuum_tests_inserted(io, store, &made.held, database, key, 100, writer); + kept = txn_vacuum_tests_inserted(io, store, &made.held, database, 3, 100, writer); + txn_vacuum_tests_committed_status(io, writer, 100, 1464, 85); + if (same) + { + txn_vacuum_tests_deleted(io, &made.held, 1, writer); + txn_vacuum_tests_deleted(io, &made.held, 2, writer); + txn_vacuum_tests_ended(io, writer, true); + enders[0] = enders[1] = 0; + } + else + { + HostXid deleter; + HostXid updater; + + txn_vacuum_tests_ended(io, writer, true); + deleter = simc_begin(); + updater = simc_begin(); + txn_vacuum_tests_deleted(io, &made.held, 1, deleter); + txn_vacuum_tests_updated(io, store, &made.held, database, 2, tids[1], 100, updater, NULL); + txn_vacuum_tests_committed_status(io, deleter, 120, 1478, 94); + txn_vacuum_tests_ended(io, deleter, true); + txn_vacuum_tests_committed_status(io, updater, 140, 1478, 94); + txn_vacuum_tests_ended(io, updater, true); + enders[0] = deleter.f0; + enders[1] = updater.f0; + } + STD_ASSERT(1485, 13, txn_vacuum_tests_folded(io, &made.repo, &made.reading, &made.held, 1485, 105), + "crate::fold::folded(io, (&repo, &reading), &held, &order, false, 200, (None, None)).unwrap()"); + takers = txn_vacuum_tests_takers(io, &made.held, __LINE__, 9); + if (takers.len != 0) + std_panic_fmt(RUST_FILE, __LINE__, 9, &message); + txn_vacuum_tests_takers_free(&takers); + if (!txn_state_base_record(io, &made.held, &some, &base, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + if (!some || !base.has_pending) + STD_OPTION_UNWRAP_FAILED(__LINE__, 9); + std_vec_new(&stored_enders, sizeof(uint64), _Alignof(uint64)); + for (size_t k = 0; k < lengthof(tids); k++) + { + uint8 key[6]; + StdVec value; + ChunksErrorError failed; + uint64 ender; + uint32 heap_bytes; + StdVec values; + bool found_value; + + txn_base_tuple_id_key(tids[k], key); + if (!trees_get(io, store, TREES_STORE_TREE_NAMED(&base.pending), key, sizeof(key), &found_value, &value, + &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!found_value) + STD_OPTION_UNWRAP_FAILED(__LINE__, 9); + if (!txn_base_pending_read(value.ptr, value.len, &ender, &heap_bytes, &values, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + std_vec_free_Checked(&values, sizeof(uint32), _Alignof(uint32)); + std_vec_free_Checked(&value, 1, 1); + std_vec_push_Checked(&stored_enders, &ender, sizeof(uint64), _Alignof(uint64)); + } + txn_base_Base_drop(&base); + kept_enders[0] = same ? writer.f0 | TXN_BASE_OWN : enders[0]; + kept_enders[1] = same ? writer.f0 | TXN_BASE_OWN : enders[1]; + txn_vacuum_tests_u64s_is_with(1496, 13, &stored_enders, kept_enders, lengthof(kept_enders), &message); + found = txn_vacuum_tests_found_every(io, store, &made.held, 1497, 81); + want[0] = (TxnVacuumFreed) + { + tids[0], 100, enders[0] + }; + want[1] = (TxnVacuumFreed) + { + tids[1], 100, enders[1] + }; + txn_vacuum_tests_found_is_with(1498, 13, &found, want, lengthof(want), &message); + for (size_t k = 0; k < found.len; k++) + any = any || STD_VEC_AT(&found, TxnVacuumFreed, k).f0 == kept; + STD_ASSERT(1499, 13, !any, "!found.iter().any(|&(tid, ..)| tid == kept)"); + txn_vacuum_tests_freed(io, store, &made.held, database, &found, false, 1500, 96); + horizons = txn_vacuum_tests_horizons(database); + txn_vacuum_tests_u64s_is_with(1508, 13, &horizons, &enders[1], 1, &message); + std_vec_free_Checked(&horizons, sizeof(uint64), _Alignof(uint64)); + txn_vacuum_tests_found_free(&found); + std_vec_free_Checked(&stored_enders, sizeof(uint64), _Alignof(uint64)); + txn_vacuum_tests_made_drop(&made); +} + +/* + * The fold's drop's D2: rows a fold's point applied and whose tuple ids its + * base doesn't pair leave the delta with the drop, and with them their + * takes, whose tuple ids the base's pending tree holds with each row's + * ender's xid (bugs row 228's (E)), so VACUUM's free of the tuple ids logs + * the newest as the block's conflict horizon, as heap's prune record holds + * the newest removed tuple's xmax: a row another transaction deleted and one + * another transaction updated give their enders, and a row its own + * transaction inserted and deleted keeps its writer's xid, marked OWN, but + * gives no horizon, so a block of only such rows logs 0. + */ +void +test_txn_vacuum_tests_a_dropped_rows_tuple_id_keeps_its_enders_xid_for_its_frees_horizon(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_enders(&test_io.io, 9457, false); + txn_vacuum_tests_enders(&test_io.io, 9458, true); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_widened is the epoch case: a take's ender just below an epoch's start widens against its writer to the earlier epoch's. */ +static void +txn_vacuum_tests_widened(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + HostXid xid = {(UINT64CONST(2) << 32) + 5}; + uint64 tid = txn_vacuum_tests_inserted(io, store, held, 9459, 1, 100, xid); + HostRow taken = {tid}; + HostError error; + StdVec takers; + StdVec records; + txn_TupleWriterRowEntryU64TupleVecU32Bool_t *last; + TxnVacuumFreed freeing; + StdVec claimed; + + if (!txn_memtable_map_ended(io, host_shared_Held_record(held), taken, xid.f0, 0xfffffff0, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1521, 66, &error); + takers = txn_vacuum_tests_takers(io, held, 1522, 93); + if (takers.len == 0) + STD_OPTION_UNWRAP_FAILED(1522, 108); + last = &STD_VEC_AT(&takers, txn_TupleWriterRowEntryU64TupleVecU32Bool_t, takers.len - 1); + { + HostRow row = {tid}; + uint64 writer = (UINT64CONST(2) << 32) + 5; + uint64 ender = (UINT64CONST(2) << 32) - 16; + StdTupleRef got = STD_TUPLE(3, STD_ITEM(&last->f0.xid, std_u64_Debug_fmt), STD_ITEM(&last->f1, host_Row_Debug_fmt), + STD_ITEM(&last->f3, std_u64_Debug_fmt)); + StdTupleRef want = STD_TUPLE(3, STD_ITEM(&writer, std_u64_Debug_fmt), STD_ITEM(&row, host_Row_Debug_fmt), + STD_ITEM(&ender, std_u64_Debug_fmt)); + + STD_ASSERT_EQ(1523, 13, last->f0.xid == writer && last->f1.f0 == tid && last->f3 == ender, &got, + std_tuple_Debug_fmt, &want, std_tuple_Debug_fmt); + } + freeing = (TxnVacuumFreed) + { + tid, 100, last->f3 + }; + std_vec_new(&freeing.f3, sizeof(uint32), _Alignof(uint32)); + claimed = txn_vacuum_tests_claimed(io, store, held, 9459, &freeing, 1, false, 1525, 18); + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + txn_vacuum_tests_takers_free(&takers); + records = txn_vacuum_tests_logged(9459); + if (records.len == 0 || !STD_VEC_AT(&records, txn_vacuum_tests_logged_t, records.len - 1).freed) + STD_PANIC(1526, 75, "no free was logged"); + STD_ASSERT_EQ_U64(1527, 13, STD_VEC_AT(&records, txn_vacuum_tests_logged_t, records.len - 1).free.horizon, + (UINT64CONST(2) << 32) - 16); + txn_vacuum_tests_logged_free(&records); + if (!txn_written_ended(io, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(1528, 39, &error); +} + +/* + * D2's width: a take keeps its row's ender's 32 bits, which read back + * widened against the take's writer's full xid, so an ender just below an + * epoch's start whose take's writer is in the next epoch reads as the earlier + * epoch's, and the free's record carries that full xid as its horizon. + */ +void +test_txn_vacuum_tests_a_takes_ender_widens_against_its_writer_across_an_epoch(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9459, txn_vacuum_tests_widened, NULL); + simc_TestIo_drop(&test_io); +} + +/* + * Bugs row 101: VACUUM's find leaves a dropped row's tuple id while the + * relation's horizon hasn't passed its ender, as heap's prune keeps a row a + * standby's feedback holds, and finds it once the horizon passes. + */ +void +test_txn_vacuum_tests_a_dropped_rows_tuple_id_waits_for_the_horizon_to_pass_its_ender(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_made_t made; + TreesStoreReader reader; + TreesStoreNodeStore store; + HostXid writer; + HostXid deleter; + HostXid updater; + uint64 tids[2]; + SimcTaken horizon; + StdVec found; + TxnVacuumFreed between[1]; + TxnVacuumFreed passed[2]; + + txn_vacuum_tests_made(io, 9461, 1539, 87, 1540, 46, 1541, 106, &made); + reader = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + store = TREES_STORE_READER_NODE_STORE(&reader); + writer = simc_begin(); + for (uint64 key = 1; key <= 2; key++) + tids[key - 1] = txn_vacuum_tests_inserted(io, store, &made.held, 9461, key, 100, writer); + txn_vacuum_tests_committed_status(io, writer, 100, 1545, 81); + txn_vacuum_tests_ended(io, writer, true); + deleter = simc_begin(); + updater = simc_begin(); + txn_vacuum_tests_deleted(io, &made.held, 1, deleter); + txn_vacuum_tests_updated(io, store, &made.held, 9461, 2, tids[1], 100, updater, NULL); + txn_vacuum_tests_committed_status(io, deleter, 120, 1551, 82); + txn_vacuum_tests_ended(io, deleter, true); + txn_vacuum_tests_committed_status(io, updater, 140, 1551, 82); + txn_vacuum_tests_ended(io, updater, true); + STD_ASSERT(1555, 9, txn_vacuum_tests_folded(io, &made.repo, &made.reading, &made.held, 1555, 101), + "crate::fold::folded(io, (&repo, &reading), &held, &order, false, 200, (None, None)).unwrap()"); + horizon = simc_non_vacuumable(deleter); + found = txn_vacuum_tests_found(io, store, &made.held, &horizon, 1556, 102); + simc_Taken_drop(&horizon); + txn_vacuum_tests_found_is(1557, 9, &found, NULL, 0); + txn_vacuum_tests_found_free(&found); + horizon = simc_non_vacuumable(updater); + found = txn_vacuum_tests_found(io, store, &made.held, &horizon, 1558, 100); + simc_Taken_drop(&horizon); + between[0] = (TxnVacuumFreed) + { + tids[0], 100, deleter.f0 + }; + txn_vacuum_tests_found_is(1559, 9, &found, between, lengthof(between)); + txn_vacuum_tests_found_free(&found); + found = txn_vacuum_tests_found_every(io, store, &made.held, 1560, 78); + passed[0] = between[0]; + passed[1] = (TxnVacuumFreed) + { + tids[1], 100, updater.f0 + }; + txn_vacuum_tests_found_is(1561, 9, &found, passed, lengthof(passed)); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* + * PRevPF 8's R101a: a row its own transaction inserted and deleted keeps its + * tuple id taken while the relation's horizon hasn't passed that transaction, + * as heap's prune keeps a tuple whose xmax equals its xmin, and its free then + * logs no conflict horizon. + */ +void +test_txn_vacuum_tests_a_row_its_own_transaction_inserted_and_deleted_waits_for_the_horizon(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_made_t made; + TreesStoreReader reader; + TreesStoreNodeStore store; + HostXid writer; + uint64 tid; + SimcTaken horizon; + StdVec found; + TxnVacuumFreed passed[1]; + + txn_vacuum_tests_made(io, 9462, 1572, 87, 1573, 46, 1574, 106, &made); + reader = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + store = TREES_STORE_READER_NODE_STORE(&reader); + writer = simc_begin(); + tid = txn_vacuum_tests_inserted(io, store, &made.held, 9462, 1, 100, writer); + txn_vacuum_tests_committed_status(io, writer, 100, 1578, 81); + txn_vacuum_tests_deleted(io, &made.held, 1, writer); + txn_vacuum_tests_ended(io, writer, true); + STD_ASSERT(1582, 9, txn_vacuum_tests_folded(io, &made.repo, &made.reading, &made.held, 1582, 101), + "crate::fold::folded(io, (&repo, &reading), &held, &order, false, 200, (None, None)).unwrap()"); + horizon = simc_non_vacuumable(writer); + found = txn_vacuum_tests_found(io, store, &made.held, &horizon, 1583, 101); + simc_Taken_drop(&horizon); + txn_vacuum_tests_found_is(1584, 9, &found, NULL, 0); + txn_vacuum_tests_found_free(&found); + found = txn_vacuum_tests_found_every(io, store, &made.held, 1585, 76); + passed[0] = (TxnVacuumFreed) + { + tid, 100, 0 + }; + txn_vacuum_tests_found_is(1585, 9, &found, passed, lengthof(passed)); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_spilled_block is the spilled block case: two takes of a block spilled, then a free of the first's tuple id. */ +static void +txn_vacuum_tests_spilled_block(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + HostStorage storage = txn_insert_tests_stored(9471); + HostXid writer = simc_begin(); + uint64 first = txn_vacuum_tests_inserted(io, store, held, 9471, 1, 100, writer); + uint64 second = txn_vacuum_tests_inserted(io, store, held, 9471, 2, 100, writer); + uint32 block = (uint32) (first >> 16); + TxnVacuumFreed freeing = {first, 100, 0}; + txn_vacuum_tests_entry_t before; + txn_vacuum_tests_entry_t want; + HostError error; + StdVec claimed; + StdVec chain; + HostDeltaHold hold; + bool spilled; + + (void) captures; + STD_ASSERT(__LINE__, 9, (uint32) (second >> 16) == block, "both takes are of one block"); + if (!txn_spill_spilled(io, &storage, TXN_WRITTEN_OF_FREE, writer.f0, &spilled, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + STD_ASSERT(__LINE__, 9, spilled, "spilled(io, &storage, Of::Free, writer).unwrap()"); + if (!host_delta_hold(io, &hold, &error) || + !txn_memtable_map_versions(&TXN_MEMTABLE_MAP_FREE_TREE_OPS, io, &hold, host_shared_Held_record(held), block, + &chain, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + STD_ASSERT(__LINE__, 9, chain.len == 0, "the block's takes left the memtable"); + txn_memtable_version_versions_drop(&chain, 0); + host_delta_Hold_drop(&hold); + before = txn_vacuum_tests_entry(io, held, block); + STD_ASSERT(__LINE__, 9, before.has, "the block's newest spilled entry reads"); + want = before; + txn_free_space_Free_give_back(&want.free, (uint16) (first & 0xffff), (uint16) (first & 0xffff), 100); + want.free.full = false; + txn_vacuum_tests_committed_status(io, writer, 100, __LINE__, 9); + txn_vacuum_tests_ended(io, writer, true); + std_vec_new(&freeing.f3, sizeof(uint32), _Alignof(uint32)); + claimed = txn_vacuum_tests_claimed(io, store, held, 9471, &freeing, 1, false, __LINE__, 9); + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + std_vec_free_Checked(&freeing.f3, sizeof(uint32), _Alignof(uint32)); + txn_vacuum_tests_entry_is(__LINE__, 9, txn_vacuum_tests_entry(io, held, block), want); +} + +/* + * VACUUM starts from a block's newest version, spilled or not: a free of a + * tuple id of a block whose free space takes all spilled gives its bytes and + * offset back to the block's newest spilled entry, never to the base's entry + * below it, so the block keeps the other take. + */ +void +test_txn_vacuum_tests_a_free_of_a_block_whose_takes_all_spilled_starts_from_its_newest_spilled_entry(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9471, txn_vacuum_tests_spilled_block, NULL); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_taken_again is the case of a freed tuple id another writer took again before the fold's drop ended its first row. */ +static void +txn_vacuum_tests_taken_again(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + HostXid first = {5}; + HostXid second = {6}; + uint64 tid = txn_vacuum_tests_inserted(io, store, held, 9472, 1, 100, first); + TxnVacuumFreed freeing = {tid, 100, 0}; + HostRow taken = {tid}; + HostError error; + StdVec claimed; + StdVec takers; + uint64 enders[2] = {0}; + + txn_vacuum_tests_ended(io, first, true); + std_vec_new(&freeing.f3, sizeof(uint32), _Alignof(uint32)); + claimed = txn_vacuum_tests_claimed(io, store, held, 9472, &freeing, 1, false, __LINE__, 12); + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_inserted(io, store, held, 9472, 2, 100, second), tid); + txn_vacuum_tests_ended(io, second, true); + if (!txn_memtable_map_ended(io, host_shared_Held_record(held), taken, first.f0, 9, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 87, &error); + takers = txn_vacuum_tests_takers(io, held, __LINE__, 11); + for (size_t i = 0; i < takers.len; i++) + { + const txn_TupleWriterRowEntryU64TupleVecU32Bool_t *taker = &STD_VEC_AT(&takers, + txn_TupleWriterRowEntryU64TupleVecU32Bool_t, + i); + + if (taker->f1.f0 == tid && taker->f0.xid == first.f0) + enders[0] = taker->f3; + else if (taker->f1.f0 == tid && taker->f0.xid == second.f0) + enders[1] = taker->f3; + } + txn_vacuum_tests_takers_free(&takers); + STD_ASSERT_EQ_U64(__LINE__, 5, enders[0], 9); + STD_ASSERT_EQ_U64(__LINE__, 5, enders[1], 0); +} + +/* + * A fold's drop stores a row's ender on the take by which the row's writer + * took its tuple id: on a standby whose fold lags its replay, VACUUM may + * already have freed the tuple id and another writer taken it again, and + * that later take keeps no ender, as on its primary, where the fold ran + * before the free. + */ +void +test_txn_vacuum_tests_a_drops_ender_goes_on_its_rows_take_not_a_later_take_of_the_tuple_id(void) +{ + SimcTestIo test_io = simc_io(); + + txn_vacuum_tests_with(&test_io.io, 9472, txn_vacuum_tests_taken_again, NULL); + simc_TestIo_drop(&test_io); +} + + +/* txn_vacuum_tests_holders returns the writers of the row versions in a tuple id's key's chain that hold it, newest first. */ +static StdVec +txn_vacuum_tests_holders(HostIoIo *io, const HostSharedHeld *held, uint64 tid) +{ + HostSharedRecord record = host_shared_Held_record(held); + HostRow row = {tid}; + HostDeltaHold hold; + HostError error; + StdVec key; + StdVec chain; + StdVec writers; + bool some; + + std_vec_new(&writers, sizeof(uint64), _Alignof(uint64)); + if (!txn_memtable_tuples_get(io, record, row, &some, &key, &error) || !some || + !host_delta_hold(io, &hold, &error) || + !txn_memtable_table_versions(io, &hold, record, key.ptr, key.len, &chain, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + for (size_t i = 0; i < chain.len; i++) + { + TxnMemtableVersionVersion *version = &STD_VEC_AT(&chain, TxnMemtableVersionVersion, i); + TxnMemtableVersionRecord stored = txn_memtable_version_Version_record(version); + uint64 xid = txn_memtable_version_Record_writer(&stored).xid; + + if (txn_vacuum_holds(version, tid)) + std_vec_push_Checked(&writers, &xid, sizeof(uint64), _Alignof(uint64)); + } + txn_memtable_version_versions_drop(&chain, 0); + host_delta_Hold_drop(&hold); + std_vec_free_Checked(&key, 1, 1); + return writers; +} + +/* + * txn_vacuum_tests_retaken is row 320's same-key case: a row ended by a + * deleter, its tuple id freed and taken again on the same key, the retake + * ended by another writer when its captures say so, then the free's take-out + * as a lagging standby defers it. + */ +static void +txn_vacuum_tests_retaken(void *captures, HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held) +{ + const bool *ended = captures; + HostXid writer = simc_begin(); + HostXid deleter = simc_begin(); + HostXid retaker = simc_begin(); + HostXid ender = simc_begin(); + uint64 tid = txn_vacuum_tests_inserted(io, store, held, 9473, 1, 100, writer); + TxnVacuumFreed freeing = {tid, 100, deleter.f0, .f4 = deleter.f0}; + std_TupleU32U16U16U64_t run = {(uint32) (tid >> 16), (uint16) tid, 1, 100}; + uint64 enders[] = {deleter.f0}; + TxnRecordTidsFreed freed = {.storage = txn_insert_tests_stored(9473), .runs = txn_vacuum_runs_view(&run, 1), + .dead = txn_vacuum_runs_view(NULL, 0), .enders = {enders, 1, 1}}; + HostError error; + StdVec claimed; + StdVec holders; + + txn_vacuum_tests_ended(io, writer, true); + txn_vacuum_tests_deleted(io, held, 1, deleter); + txn_vacuum_tests_ended(io, deleter, true); + std_vec_new(&freeing.f3, sizeof(uint32), _Alignof(uint32)); + claimed = txn_vacuum_tests_claimed(io, store, held, 9473, &freeing, 1, false, __LINE__, 12); + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_inserted(io, store, held, 9473, 1, 100, retaker), tid); + txn_vacuum_tests_ended(io, retaker, true); + if (*ended) + { + txn_vacuum_tests_deleted(io, held, 1, ender); + txn_vacuum_tests_ended(io, ender, true); + } + holders = txn_vacuum_tests_holders(io, held, tid); + STD_ASSERT_EQ_U64(__LINE__, 5, holders.len, 2); + std_vec_free_Checked(&holders, sizeof(uint64), _Alignof(uint64)); + if (!txn_vacuum_taken_out(io, held, &freed, 1, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + holders = txn_vacuum_tests_holders(io, held, tid); + STD_ASSERT_EQ_U64(__LINE__, 5, holders.len, 1); + STD_ASSERT_EQ_U64(__LINE__, 5, STD_VEC_AT(&holders, uint64, 0), retaker.f0); + std_vec_free_Checked(&holders, sizeof(uint64), _Alignof(uint64)); +} + +/* + * Bugs row 320's same-key case: a lagging standby's deferred take-out of a + * free whose tuple id was taken again on the same key takes out the freed + * incarnation's version, the one its stored ender ended, and leaves the + * retake's, live or ended by another writer, as its primary does. + */ +void +test_txn_vacuum_tests_a_frees_take_out_leaves_a_retake_on_its_key(void) +{ + SimcTestIo test_io = simc_io(); + bool live = false; + bool ended = true; + + txn_vacuum_tests_with(&test_io.io, 9473, txn_vacuum_tests_retaken, &live); + simc_TestIo_drop(&test_io); + test_io = simc_io(); + txn_vacuum_tests_with(&test_io.io, 9473, txn_vacuum_tests_retaken, &ended); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_holder checks that a tuple id's key's chain has one version that holds it, its writer's. */ +static void +txn_vacuum_tests_holder(HostIoIo *io, const HostSharedHeld *held, uint64 tid, HostXid writer, uint32 line) +{ + StdVec holders = txn_vacuum_tests_holders(io, held, tid); + + STD_ASSERT_EQ_U64(line, 5, holders.len, 1); + STD_ASSERT_EQ_U64(line, 5, STD_VEC_AT(&holders, uint64, 0), writer.f0); + std_vec_free_Checked(&holders, sizeof(uint64), _Alignof(uint64)); +} + +/* txn_vacuum_tests_taken_out_as takes a free of one tuple id with its stored ender out of a storage's delta, as a lagging standby defers it, in field 6 when dead. */ +static void +txn_vacuum_tests_taken_out_as(HostIoIo *io, const HostSharedHeld *held, uint32 database, uint64 tid, uint64 ender, + bool dead) +{ + std_TupleU32U16U16U64_t run = {(uint32) (tid >> 16), (uint16) tid, 1, 100}; + TxnRecordTidsFreed freed = {.storage = txn_insert_tests_stored(database), .runs = txn_vacuum_runs_view(&run, 1), + .dead = txn_vacuum_runs_view(NULL, 0), .enders = {&ender, 1, 1}}; + HostError error; + + if (dead) + { + freed.dead = freed.runs; + freed.runs = txn_vacuum_runs_view(NULL, 0); + } + if (!txn_vacuum_taken_out(io, held, &freed, 1, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); +} + +/* TXN_VACUUM_TESTS_RETAKE_RUNNING and the two after it are how the no-ender case's retake ends: left running, committed, or aborted. */ +#define TXN_VACUUM_TESTS_RETAKE_RUNNING 0 +#define TXN_VACUUM_TESTS_RETAKE_COMMITTED 1 +#define TXN_VACUUM_TESTS_RETAKE_ABORTED 2 + +/* + * txn_vacuum_tests_unended is row 320's no-ender case: an aborted insert a + * fold dropped beside another key's committed row, so its free finds no + * ender, its tuple id taken again on the same key by a retake that ends as + * the case says, then the free's take-out, in field 5 or as a dead run in + * field 6, as a lagging standby defers it; a retake aborted as a standby + * replays its abort keeps its version until a later fold, of another key's + * committed row, drops it. + */ +static void +txn_vacuum_tests_unended(int retake, bool dead) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_made_t made; + TreesStoreReader reader; + TreesStoreNodeStore store; + HostXid writer = simc_begin(); + HostXid keeper = simc_begin(); + HostXid retaker; + uint64 tid; + uint64 ender; + StdVec found; + StdVec claimed; + + txn_vacuum_tests_made(io, 9474, __LINE__, 2, __LINE__, 2, __LINE__, 2, &made); + reader = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + store = TREES_STORE_READER_NODE_STORE(&reader); + tid = txn_vacuum_tests_inserted(io, store, &made.held, 9474, 1, 100, writer); + txn_vacuum_tests_ended(io, writer, false); + (void) txn_vacuum_tests_inserted(io, store, &made.held, 9474, 2, 100, keeper); + txn_vacuum_tests_committed_at(io, keeper, 110); + txn_vacuum_tests_folded_at(io, &made, 120); + found = txn_vacuum_tests_found_every(io, store, &made.held, __LINE__, 9); + STD_ASSERT_EQ_U64(__LINE__, 5, found.len, 1); + STD_ASSERT_EQ_U64(__LINE__, 5, STD_VEC_AT(&found, TxnVacuumFreed, 0).f0, tid); + ender = STD_VEC_AT(&found, TxnVacuumFreed, 0).f4; + STD_ASSERT_EQ_U64(__LINE__, 5, ender, 0); + claimed = txn_vacuum_tests_claimed(io, store, &made.held, 9474, found.ptr, found.len, false, __LINE__, 12); + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + txn_vacuum_tests_found_free(&found); + retaker = simc_begin(); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_inserted(io, store, &made.held, 9474, 1, 100, retaker), tid); + if (retake == TXN_VACUUM_TESTS_RETAKE_ABORTED) + { + HostXid later = simc_begin(); + + simc_abort(retaker); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_versions_len(io, &made.held, 1, __LINE__, 9, __LINE__, 9), 1); + txn_vacuum_tests_taken_out_as(io, &made.held, 9474, tid, ender, dead); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_versions_len(io, &made.held, 1, __LINE__, 9, __LINE__, 9), 1); + (void) txn_vacuum_tests_inserted(io, store, &made.held, 9474, 3, 100, later); + txn_vacuum_tests_committed_at(io, later, 150); + txn_vacuum_tests_folded_at(io, &made, 200); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_versions_len(io, &made.held, 1, __LINE__, 9, __LINE__, 9), 0); + } + else + { + if (retake == TXN_VACUUM_TESTS_RETAKE_COMMITTED) + txn_vacuum_tests_committed_at(io, retaker, 130); + txn_vacuum_tests_holder(io, &made.held, tid, retaker, __LINE__); + txn_vacuum_tests_taken_out_as(io, &made.held, 9474, tid, ender, dead); + txn_vacuum_tests_holder(io, &made.held, tid, retaker, __LINE__); + if (retake == TXN_VACUUM_TESTS_RETAKE_RUNNING) + txn_vacuum_tests_committed_at(io, retaker, 130); + } + txn_vacuum_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* + * Bugs row 320's no-ender case: a free whose tuple id's row a fold already + * dropped stores no ender, and its deferred take-out on a lagging standby + * takes out nothing, in field 5 or as a skipped index pass's dead run in + * field 6, so a retake of the tuple id on the same key stays, left running, + * committed, or aborted until a fold drops it, as on its primary. + */ +void +test_txn_vacuum_tests_a_frees_take_out_with_no_ender_leaves_any_retake_on_its_key(void) +{ + for (int retake = TXN_VACUUM_TESTS_RETAKE_RUNNING; retake <= TXN_VACUUM_TESTS_RETAKE_ABORTED; retake++) + { + txn_vacuum_tests_unended(retake, false); + txn_vacuum_tests_unended(retake, true); + } +} + +/* + * Bugs row 320's subtransaction case: a row deleted in a subtransaction + * that SAVEPOINT began and RELEASE ended has its subtransaction's xid for + * the ender its primary's free finds after a fold, and the free's deferred + * take-out on a lagging standby, which holds the same changes unfolded, + * takes out the freed incarnation's version by it and leaves a retake on + * the same key. + */ +void +test_txn_vacuum_tests_a_frees_take_out_takes_out_a_subtransactions_delete(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_made_t primary; + txn_vacuum_tests_made_t standby; + TreesStoreReader primary_reader; + TreesStoreReader standby_reader; + TreesStoreNodeStore primary_store; + TreesStoreNodeStore standby_store; + HostXid writer = simc_begin(); + HostXid deleter; + HostXid released; + HostXid retaker; + uint64 tid; + uint64 ender; + StdVec found; + StdVec holders; + + txn_vacuum_tests_made(io, 9475, __LINE__, 2, __LINE__, 2, __LINE__, 2, &primary); + txn_vacuum_tests_made(io, 9476, __LINE__, 2, __LINE__, 2, __LINE__, 2, &standby); + primary_reader = repo_Repo_reader(&primary.repo, HOST_CACHE_PLACEMENT_SERVER, &primary.reading); + standby_reader = repo_Repo_reader(&standby.repo, HOST_CACHE_PLACEMENT_SERVER, &standby.reading); + primary_store = TREES_STORE_READER_NODE_STORE(&primary_reader); + standby_store = TREES_STORE_READER_NODE_STORE(&standby_reader); + tid = txn_vacuum_tests_inserted(io, primary_store, &primary.held, 9475, 1, 100, writer); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_inserted(io, standby_store, &standby.held, 9476, 1, 100, writer), + tid); + txn_vacuum_tests_committed_at(io, writer, 100); + deleter = simc_begin(); + released = simc_begin_under(deleter); + txn_vacuum_tests_deleted(io, &primary.held, 1, released); + txn_vacuum_tests_deleted(io, &standby.held, 1, released); + txn_vacuum_tests_committed_status(io, released, 110, __LINE__, 5); + simc_commit(released); + txn_vacuum_tests_committed_at(io, deleter, 110); + txn_vacuum_tests_folded_at(io, &primary, 120); + found = txn_vacuum_tests_found_every(io, primary_store, &primary.held, __LINE__, 9); + STD_ASSERT_EQ_U64(__LINE__, 5, found.len, 1); + STD_ASSERT_EQ_U64(__LINE__, 5, STD_VEC_AT(&found, TxnVacuumFreed, 0).f0, tid); + ender = STD_VEC_AT(&found, TxnVacuumFreed, 0).f4; + STD_ASSERT_EQ_U64(__LINE__, 5, ender, released.f0); + txn_vacuum_tests_found_free(&found); + { + TxnVacuumFreed freeing = {tid, 100, ender, .f4 = ender}; + StdVec claimed; + + std_vec_new(&freeing.f3, sizeof(uint32), _Alignof(uint32)); + claimed = txn_vacuum_tests_claimed(io, standby_store, &standby.held, 9476, &freeing, 1, false, __LINE__, 12); + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + } + retaker = simc_begin(); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_inserted(io, standby_store, &standby.held, 9476, 1, 100, retaker), + tid); + txn_vacuum_tests_committed_at(io, retaker, 130); + holders = txn_vacuum_tests_holders(io, &standby.held, tid); + STD_ASSERT_EQ_U64(__LINE__, 5, holders.len, 2); + std_vec_free_Checked(&holders, sizeof(uint64), _Alignof(uint64)); + txn_vacuum_tests_taken_out_as(io, &standby.held, 9476, tid, ender, false); + txn_vacuum_tests_holder(io, &standby.held, tid, retaker, __LINE__); + txn_vacuum_tests_made_drop(&standby); + txn_vacuum_tests_made_drop(&primary); + simc_TestIo_drop(&test_io); +} + +/* txn_vacuum_tests_enders_body returns a free's body of two tuple ids in field 5 and one dead in field 6, with the enders given. */ +static StdVec +txn_vacuum_tests_enders_body(const uint64 *enders, size_t enders_len) +{ + static const std_TupleU32U16U16U64_t runs[] = {{0, 1, 2, 200}}; + static const std_TupleU32U16U16U64_t dead[] = {{0, 5, 1, 100}}; + TxnRecordTidsFreed freed = {.storage = txn_insert_tests_stored(9477), .horizon = 9, + .runs = txn_vacuum_runs_view(runs, lengthof(runs)), .dead = txn_vacuum_runs_view(dead, lengthof(dead)), + .enders = {(void *) enders, enders_len, enders_len}}; + StdVec body; + + txn_record_TidsFreed_encode(&freed, &body); + return body; +} + +/* txn_vacuum_tests_rewritten returns a free's body written again field by field with field 10 replaced by the bytes given, or left out for none. */ +static StdVec +txn_vacuum_tests_rewritten(const StdVec *body, const uint8 *enders, size_t enders_len) +{ + ChunksRecordRecord record; + ChunksRecordRecordWriter writer = chunks_record_RecordWriter_new(); + ChunksErrorError failed; + StdVec rewritten; + + if (!chunks_record_Record_value(body->ptr, body->len, NULL, 0, &record, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + for (uint32 number = 1; number <= 4; number++) + { + uint64 value; + + if (!chunks_record_Record_uint(&record, number, &value, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + chunks_record_RecordWriter_uint(&writer, number, value); + } + for (uint32 number = 5; number <= 6; number++) + { + size_t len; + const uint8 *bytes = chunks_record_Record_bytes(&record, number, &len); + + if (bytes != NULL) + chunks_record_RecordWriter_field(&writer, number, bytes, len); + } + if (enders != NULL) + chunks_record_RecordWriter_field(&writer, 10, enders, enders_len); + rewritten = chunks_record_RecordWriter_finish(&writer); + chunks_record_RecordWriter_drop(&writer); + chunks_record_Record_drop(&record); + return rewritten; +} + +/* txn_vacuum_tests_decoded decodes a free's body, checking that it reads back as the free given, or with none given, that it is refused as data corruption. */ +static void +txn_vacuum_tests_decoded(StdVec body, const TxnRecordTidsFreed *want, uint32 line) +{ + TxnRecordTidsFreed decoded; + HostError error; + bool read = txn_record_TidsFreed_decode(body.ptr, body.len, TXN_VACUUM_TESTS_PAGE.most, + TXN_VACUUM_TESTS_PAGE.bytes, &decoded, &error); + + if (want != NULL) + { + if (!read) + TXN_VACUUM_TESTS_UNWRAP_FAILED(line, 9, &error); + STD_ASSERT(line, 5, txn_record_TidsFreed_PartialEq_eq(&decoded, want), "the free reads back"); + txn_record_TidsFreed_drop(&decoded); + } + else + { + STD_ASSERT(line, 5, !read, "the body is refused"); + STD_ASSERT(line, 5, memcmp(host_Error_sqlstate(&error), "XX001", 5) == 0, "as data corruption"); + host_Error_drop(&error); + } + std_vec_free_Checked(&body, 1, 1); +} + +/* + * Bugs row 320's field 10: a free's enders, one for each tuple id of field + * 5 then of field 6, read back with an ender of 0 in either field, and a + * body written again field by field reads back the same; fewer or more + * enders than tuple ids, field 10 left out of a free with runs or written + * into one with none, and a cut varint are each refused as data corruption. + */ +void +test_txn_vacuum_tests_a_frees_enders_of_another_form_are_refused_as_corruption(void) +{ + static const std_TupleU32U16U16U64_t runs[] = {{0, 1, 2, 200}}; + static const std_TupleU32U16U16U64_t dead[] = {{0, 5, 1, 100}}; + static const uint64 enders[] = {7, 0, 0}; + static const uint64 more[] = {7, 0, 0, 4}; + static const uint8 one[] = {7}; + static const uint8 cut[] = {7, 0, 0x89}; + TxnRecordTidsFreed want = {.storage = txn_insert_tests_stored(9477), .horizon = 9, + .runs = txn_vacuum_runs_view(runs, lengthof(runs)), .dead = txn_vacuum_runs_view(dead, lengthof(dead)), + .enders = {(void *) enders, lengthof(enders), lengthof(enders)}}; + TxnRecordTidsFreed none = {.storage = txn_insert_tests_stored(9477), .horizon = 9, + .runs = txn_vacuum_runs_view(NULL, 0), .dead = txn_vacuum_runs_view(NULL, 0), + .enders = {NULL, 0, 0}}; + StdVec body = txn_vacuum_tests_enders_body(enders, lengthof(enders)); + StdVec empty; + StdVec own; + const uint8 *written; + size_t written_len; + + { + ChunksRecordRecord record; + ChunksErrorError failed; + + if (!chunks_record_Record_value(body.ptr, body.len, NULL, 0, &record, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 9, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + written = chunks_record_Record_bytes(&record, 10, &written_len); + STD_ASSERT(__LINE__, 5, written != NULL, "field 10 is written"); + own = txn_vacuum_tests_rewritten(&body, written, written_len); + chunks_record_Record_drop(&record); + } + txn_vacuum_tests_decoded(own, &want, __LINE__); + txn_vacuum_tests_decoded(txn_vacuum_tests_rewritten(&body, NULL, 0), NULL, __LINE__); + txn_vacuum_tests_decoded(txn_vacuum_tests_rewritten(&body, cut, lengthof(cut)), NULL, __LINE__); + txn_vacuum_tests_decoded(body, &want, __LINE__); + txn_vacuum_tests_decoded(txn_vacuum_tests_enders_body(enders, 2), NULL, __LINE__); + txn_vacuum_tests_decoded(txn_vacuum_tests_enders_body(more, lengthof(more)), NULL, __LINE__); + txn_record_TidsFreed_encode(&none, &empty); + txn_vacuum_tests_decoded(txn_vacuum_tests_rewritten(&empty, NULL, 0), &none, __LINE__); + txn_vacuum_tests_decoded(txn_vacuum_tests_rewritten(&empty, one, lengthof(one)), NULL, __LINE__); + std_vec_free_Checked(&empty, 1, 1); +} + +/* txn_vacuum_tests_pruned_block prunes block 0 of a case's storage past every xid, as a take on it does when its free bytes are below the reserve. */ +static void +txn_vacuum_tests_pruned_block(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 database) +{ + SimcTaken horizon = txn_vacuum_tests_every(); + SimcTaken any = simc_snapshot(HOST_SNAPSHOT_KIND_ANY, false); + HostSnapshot horizon_lent = simc_Taken_lent(&horizon); + HostSnapshot any_lent = simc_Taken_lent(&any); + TxnVacuumPrune prune = {.horizon = &horizon_lent, .any = &any_lent}; + HostError error; + + if (!txn_vacuum_pruned(io, store, held, txn_insert_tests_stored(database), true, &TXN_VACUUM_TESTS_PAGE, 0, + TXN_VACUUM_TESTS_PAGE.bytes, &prune, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + simc_Taken_drop(&any); + simc_Taken_drop(&horizon); +} + +/* + * Bugs row 320's prune case: a row an aborted writer inserted and deleted + * itself is claimed dead, before the abort's cleanup takes it out, by a + * prune whose record names its ender with OWN; a lagging standby holding the + * same changes marks it dead at its own record's LSN, frees it with the + * ender its primary's VACUUM finds, and takes it again on the same key, and + * both records' deferred take-outs, the prune's and the free's, take out + * only the aborted writer's version by its writer and leave the retake. + */ +void +test_txn_vacuum_tests_a_prunes_take_out_of_an_aborted_writers_own_row_leaves_a_retake(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_made_t primary; + txn_vacuum_tests_made_t standby; + TreesStoreReader primary_reader; + TreesStoreReader standby_reader; + TreesStoreNodeStore primary_store; + TreesStoreNodeStore standby_store; + HostXid aborted = simc_begin(); + HostXid retaker = simc_begin(); + const TxnRecordTidsFreed *pruned = NULL; + uint64 tid; + StdVec records; + StdVec found; + StdVec marked; + HostLsn marking_end = 0; + TxnVacuumFreed freeing; + HostError error; + + txn_vacuum_tests_made(io, 9478, __LINE__, 2, __LINE__, 2, __LINE__, 2, &primary); + txn_vacuum_tests_made(io, 9479, __LINE__, 2, __LINE__, 2, __LINE__, 2, &standby); + primary_reader = repo_Repo_reader(&primary.repo, HOST_CACHE_PLACEMENT_SERVER, &primary.reading); + standby_reader = repo_Repo_reader(&standby.repo, HOST_CACHE_PLACEMENT_SERVER, &standby.reading); + primary_store = TREES_STORE_READER_NODE_STORE(&primary_reader); + standby_store = TREES_STORE_READER_NODE_STORE(&standby_reader); + tid = txn_vacuum_tests_inserted(io, primary_store, &primary.held, 9478, 1, 100, aborted); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_inserted(io, standby_store, &standby.held, 9479, 1, 100, aborted), + tid); + txn_vacuum_tests_deleted(io, &primary.held, 1, aborted); + txn_vacuum_tests_deleted(io, &standby.held, 1, aborted); + simc_abort(aborted); + txn_vacuum_tests_pruned_block(io, primary_store, &primary.held, 9478); + records = txn_vacuum_tests_logged(9478); + for (size_t k = 0; k < records.len; k++) + if (STD_VEC_AT(&records, txn_vacuum_tests_logged_t, k).freed) + pruned = &STD_VEC_AT(&records, txn_vacuum_tests_logged_t, k).free; + STD_ASSERT(__LINE__, 5, pruned != NULL, "the prune logs its free"); + STD_ASSERT_EQ_U64(__LINE__, 5, pruned->runs.len, 0); + STD_ASSERT_EQ_U64(__LINE__, 5, pruned->dead.len, 1); + STD_ASSERT_EQ_U64(__LINE__, 5, pruned->enders.len, 1); + STD_ASSERT_EQ_U64(__LINE__, 5, STD_VEC_AT(&pruned->enders, uint64, 0), aborted.f0 | TXN_BASE_OWN); + found = txn_vacuum_tests_found_every(io, primary_store, &primary.held, __LINE__, 9); + STD_ASSERT_EQ_U64(__LINE__, 5, found.len, 1); + freeing = STD_VEC_AT(&found, TxnVacuumFreed, 0); + STD_ASSERT_EQ_U64(__LINE__, 5, freeing.f0, tid); + { + TxnVacuumFreed marking = {tid, 100, 0, .f4 = aborted.f0 | TXN_BASE_OWN}; + TxnVacuumFreed freed = {tid, freeing.f1, freeing.f2, .f4 = freeing.f4}; + StdVec claimed; + + std_vec_new(&marking.f3, sizeof(uint32), _Alignof(uint32)); + std_vec_new(&freed.f3, sizeof(uint32), _Alignof(uint32)); + claimed = txn_vacuum_tests_claimed(io, standby_store, &standby.held, 9479, &marking, 1, true, __LINE__, 12); + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + marked = txn_vacuum_tests_logged(9479); + for (size_t k = 0; k < marked.len; k++) + if (STD_VEC_AT(&marked, txn_vacuum_tests_logged_t, k).freed) + marking_end = STD_VEC_AT(&marked, txn_vacuum_tests_logged_t, k).end; + STD_ASSERT(__LINE__, 5, marking_end != 0, "the marking logs its free"); + claimed = txn_vacuum_tests_claimed(io, standby_store, &standby.held, 9479, &freed, 1, false, __LINE__, 12); + std_vec_free_Checked(&claimed, sizeof(uint64), _Alignof(uint64)); + } + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_inserted(io, standby_store, &standby.held, 9479, 1, 100, retaker), + tid); + txn_vacuum_tests_committed_at(io, retaker, 130); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_versions_len(io, &standby.held, 1, __LINE__, 9, __LINE__, 9), 3); + { + TxnRecordTidsFreed taken = *pruned; + + taken.storage = txn_insert_tests_stored(9479); + if (!txn_vacuum_taken_out(io, &standby.held, &taken, marking_end, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + } + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_versions_len(io, &standby.held, 1, __LINE__, 9, __LINE__, 9), 2); + txn_vacuum_tests_holder(io, &standby.held, tid, retaker, __LINE__); + txn_vacuum_tests_taken_out_as(io, &standby.held, 9479, tid, freeing.f4, false); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_versions_len(io, &standby.held, 1, __LINE__, 9, __LINE__, 9), 2); + txn_vacuum_tests_holder(io, &standby.held, tid, retaker, __LINE__); + if (!txn_written_aborted(io, 0, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + txn_vacuum_tests_found_free(&found); + txn_vacuum_tests_logged_free(&marked); + txn_vacuum_tests_logged_free(&records); + txn_vacuum_tests_made_drop(&standby); + txn_vacuum_tests_made_drop(&primary); + simc_TestIo_drop(&test_io); +} + +/* + * Bugs row 320's aborted insert: a prune over a block whose only row an + * aborted writer inserted, unfolded and before the abort's cleanup takes it + * out, claims nothing, since only a changer's end makes a tuple id dead, and + * logs no free. + */ +void +test_txn_vacuum_tests_a_prune_over_an_aborted_unfolded_insert_claims_nothing(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_vacuum_tests_made_t made; + TreesStoreReader reader; + TreesStoreNodeStore store; + HostXid aborted = simc_begin(); + StdVec records; + HostError error; + + txn_vacuum_tests_made(io, 9480, __LINE__, 2, __LINE__, 2, __LINE__, 2, &made); + reader = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + store = TREES_STORE_READER_NODE_STORE(&reader); + (void) txn_vacuum_tests_inserted(io, store, &made.held, 9480, 1, 100, aborted); + simc_abort(aborted); + txn_vacuum_tests_pruned_block(io, store, &made.held, 9480); + records = txn_vacuum_tests_logged(9480); + for (size_t k = 0; k < records.len; k++) + STD_ASSERT(__LINE__, 5, !STD_VEC_AT(&records, txn_vacuum_tests_logged_t, k).freed, "the prune logs no free"); + STD_ASSERT_EQ_U64(__LINE__, 5, txn_vacuum_tests_versions_len(io, &made.held, 1, __LINE__, 9, __LINE__, 9), 1); + if (!txn_written_aborted(io, 0, &error)) + TXN_VACUUM_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + txn_vacuum_tests_logged_free(&records); + txn_vacuum_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +#endif diff --git a/src/doltgres/txn/c/values.c b/src/doltgres/txn/c/values.c index ead873706758a..90c1a2e459e27 100644 --- a/src/doltgres/txn/c/values.c +++ b/src/doltgres/txn/c/values.c @@ -35,6 +35,22 @@ #include "doltgres/txn/state.h" #include "doltgres/txn/values.h" +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/repo/lib.h" +#include "doltgres/simc/simc.h" +#include "doltgres/std/test.h" + +/* The map's takes that keep nothing, whose prepare's closure returns nothing. */ +#define TXN_MEMTABLE_MAP_RS Unit +#define TXN_MEMTABLE_MAP_RETURNS bool +#define TXN_MEMTABLE_MAP_RESULT_PARAMS , HostError *error +#define TXN_MEMTABLE_MAP_TS Unit +#define TXN_MEMTABLE_MAP_KEPT_OUT +#define TXN_MEMTABLE_MAP_KEPT_IN +#define TXN_MEMTABLE_MAP_DECLARE +#include "doltgres/txn/memtable/map_template.h" +#endif + #define RUST_FILE "txn" STD_SEP "src" STD_SEP "values.rs" /* txn_values_maxaligned rounds a length up to Postgres' MAXALIGN, eight bytes. */ @@ -405,3 +421,319 @@ txn_values_entry(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld * txn_values_entry_pad(&frame); return frame.decoded; } + +#ifdef DOLTGRES_TEST_TXN + +/* TXN_VALUES_TESTS_UNWRAP_FAILED is a Result's unwrap's panic on a HostError at a line and column. */ +#define TXN_VALUES_TESTS_UNWRAP_FAILED(line, col, error) \ + STD_RESULT_EXPECT_FAILED((line), (col), "called `Result::unwrap()` on an `Err` value", (error), \ + host_Error_Debug_fmt) + +/* txn_values_tests_refused reports whether a value fails to decode as an entry with data corruption, XX001. */ +static bool +txn_values_tests_refused(const uint8 *value, size_t value_len) +{ + TxnValuesEntry entry; + HostError error; + bool refused; + + if (txn_values_Entry_decode(value, value_len, &entry, &error)) + return false; + refused = memcmp(host_Error_sqlstate(&error), "XX001", 5) == 0; + host_Error_drop(&error); + return refused; +} + +/* + * test_txn_values_tests_a_value_map_entry_reads_back_and_refuses_every_other_form: + * an entry reads back as it was written, in MOST's room without growing its + * buffer, and one cut short, one with a byte past its count, and one whose + * count is zero are data corruption. + */ +void +test_txn_values_tests_a_value_map_entry_reads_back_and_refuses_every_other_form(void) +{ + TxnValuesEntry entry = {.address = chunks_address_Address_of((const uint8 *) "blob", 4), + .locator = chunks_trailer_Locator_at(3, 640, 64), .raw = 4000, .stored = 4000, .count = 1}; + TxnValuesEntry zero = entry; + TxnValuesEntry decoded; + StdVec out; + StdVec longer; + StdVec none; + const void *at; + uint8 key[4]; + uint8 past = 0; + HostError error; + + std_vec_with_capacity_Checked(&out, TXN_VALUES_MOST, 1, 1); + at = out.ptr; + txn_values_Entry_encode_into(&entry, &out); + STD_ASSERT(__LINE__, 9, out.ptr == at, "an entry fits MOST's room"); + if (!txn_values_Entry_decode(out.ptr, out.len, &decoded, &error)) + TXN_VALUES_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + STD_ASSERT_EQ(__LINE__, 9, txn_values_Entry_PartialEq_eq(&decoded, &entry), &decoded, txn_values_Entry_Debug_fmt, + &entry, txn_values_Entry_Debug_fmt); + txn_values_key(258, key); + STD_ASSERT(__LINE__, 9, key[0] == 0 && key[1] == 0 && key[2] == 1 && key[3] == 2, "a key is its id big-endian"); + STD_ASSERT(__LINE__, 9, txn_values_tests_refused(out.ptr, out.len - 1), "a cut entry decoded"); + std_slice_to_vec_Checked(&longer, out.ptr, out.len, 1, 1); + std_vec_push_Checked(&longer, &past, 1, 1); + STD_ASSERT(__LINE__, 9, txn_values_tests_refused(longer.ptr, longer.len), + "an entry with a byte past its count decoded"); + STD_ASSERT(__LINE__, 9, txn_values_tests_refused(out.ptr, 20), "an entry with half an address decoded"); + zero.count = 0; + std_vec_new(&none, 1, 1); + txn_values_Entry_encode_into(&zero, &none); + STD_ASSERT(__LINE__, 9, txn_values_tests_refused(none.ptr, none.len), "a count of zero decoded"); + std_vec_free_Checked(&none, 1, 1); + std_vec_free_Checked(&longer, 1, 1); + std_vec_free_Checked(&out, 1, 1); +} + +/* txn_values_tests_made_t is a TOAST relation's state in a new repository, with the reader its store reads through. */ +typedef struct txn_values_tests_made_t +{ + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld toast; +} txn_values_tests_made_t; + +/* txn_values_tests_made returns a new database's TOAST relation's state of a relation in pg_default. */ +static txn_values_tests_made_t +txn_values_tests_made(HostIoIo *io, uint32 database, uint32 relation) +{ + HostStorage storage = {.tablespace = HOST_PG_DEFAULT, .database = database, .relation = relation}; + txn_values_tests_made_t made; + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_create(io, database, HOST_PG_DEFAULT, &made.repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!host_shared_read(io, &made.reading, &error) || + !txn_state_state(io, &made.repo, &made.reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, + &made.toast, &error)) + TXN_VALUES_TESTS_UNWRAP_FAILED(__LINE__, 0, &error); + return made; +} + +/* txn_values_tests_made_drop frees what made returned. */ +static void +txn_values_tests_made_drop(txn_values_tests_made_t *made) +{ + host_shared_Held_drop(&made->toast); + host_shared_Reader_drop(&made->reading); + repo_Repo_drop(&made->repo); +} + +/* txn_values_tests_put is a take's step that puts the entry its captures hold. */ +static TxnMemtableMapStep +txn_values_tests_put(void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out) +{ + txn_values_Entry_encode_into(captures, out); + return TXN_MEMTABLE_MAP_STEP_PUT; +} + +/* txn_values_tests_nothing is a take's then that does nothing. */ +static void +txn_values_tests_nothing(void *captures) +{ +} + +/* txn_values_tests_written_t is written's closure's captures: the writer and the entry it puts. */ +typedef struct txn_values_tests_written_t +{ + TxnMemtableVersionWriter writer; + TxnValuesEntry *entry; +} txn_values_tests_written_t; + +/* txn_values_tests_written_inside is written's prepare's closure: the one take's finish. */ +static bool +txn_values_tests_written_inside(void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, + HostError *error) +{ + txn_values_tests_written_t *c = captures; + TxnStatusRoomed unroomed = {0}; + TxnMemtableMapTaken taken; + + return txn_memtable_map_finish_Unit(HOST_IO_IO_TAKES_SHORT(io), prepared, 0, c->writer, false, unroomed, false, 0, + false, NULL, 0, NULL, txn_values_tests_put, c->entry, NULL, + txn_values_tests_nothing, NULL, NULL, &taken, error); +} + +/* txn_values_tests_written puts a value id's entry into a TOAST relation's value map by a writer. */ +static void +txn_values_tests_written(HostIoIo *io, const HostSharedHeld *toast, uint32 id, uint64 xid, TxnValuesEntry *entry) +{ + TxnMemtableMapTake take = {.state = host_shared_Held_record(toast), .map = TXN_MEMTABLE_MAP_MAP_VALUES, + .key = id, .most = TXN_VALUES_MOST}; + txn_values_tests_written_t c = {.writer = {.xid = xid}, .entry = entry}; + HostError error; + + if (!txn_memtable_map_prepare_Unit(io, &take, 1, txn_values_tests_written_inside, &c, NULL, &error)) + TXN_VALUES_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); +} + +/* + * test_txn_values_tests_a_value_id_the_map_holds_is_skipped: an id the TOAST + * relation's value map holds in its delta is skipped, as GetNewOidWithIndex + * skips one the TOAST index holds, and the next id is taken. + */ +void +test_txn_values_tests_a_value_id_the_map_holds_is_skipped(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_values_tests_made_t made = txn_values_tests_made(io, 9310, 16390); + TreesStoreReader store = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + TxnValuesEntry entry = {.address = chunks_address_Address_of((const uint8 *) "v", 1), + .locator = chunks_trailer_Locator_Default_default(), .raw = 4000, .stored = 4000, .count = 1}; + bool held; + bool next; + HostError error; + + txn_values_tests_written(io, &made.toast, 100, 41, &entry); + if (!txn_values_free(io, TREES_STORE_READER_NODE_STORE(&store), &made.toast, 100, &held, &error) || + !txn_values_free(io, TREES_STORE_READER_NODE_STORE(&store), &made.toast, 101, &next, &error)) + TXN_VALUES_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + STD_ASSERT(__LINE__, 9, !held, "an id the map holds is not free"); + STD_ASSERT(__LINE__, 9, next, "the next id is free"); + txn_values_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_values_tests_a_value_ids_entry_reads_back: a value id's entry the + * delta holds reads back as it was written, and an id the map doesn't hold + * has none. + */ +void +test_txn_values_tests_a_value_ids_entry_reads_back(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + txn_values_tests_made_t made = txn_values_tests_made(io, 9311, 16391); + TreesStoreReader store = repo_Repo_reader(&made.repo, HOST_CACHE_PLACEMENT_SERVER, &made.reading); + TxnValuesEntry written = txn_values_Entry_new(chunks_address_Address_of((const uint8 *) "value", 5), 5000); + TxnValuesEntry read; + TxnValuesEntry absent; + bool has_read; + bool has_absent; + HostError error; + + txn_values_tests_written(io, &made.toast, 200, 42, &written); + if (!txn_values_entry(io, TREES_STORE_READER_NODE_STORE(&store), &made.toast, 200, &has_read, &read, &error) || + !txn_values_entry(io, TREES_STORE_READER_NODE_STORE(&store), &made.toast, 201, &has_absent, &absent, + &error)) + TXN_VALUES_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + STD_ASSERT(__LINE__, 9, has_read && txn_values_Entry_PartialEq_eq(&read, &written), + "an id's entry reads back as written"); + STD_ASSERT(__LINE__, 9, !has_absent, "an id the map doesn't hold has no entry"); + txn_values_tests_made_drop(&made); + simc_TestIo_drop(&test_io); +} + +/* txn_values_tests_sized returns a value's sizes by its stored bytes: its entries, its rows, and its heap bytes. */ +static TxnBaseSizes +txn_values_tests_sized(uint64 stored) +{ + TxnValuesEntry entry = {.address = chunks_address_Address_of((const uint8 *) "blob", 4), + .locator = chunks_trailer_Locator_Default_default(), .raw = stored + 4, .stored = stored, .count = 1}; + StdVec value; + uint8 key[4]; + TxnBaseSizes sizes; + HostError error; + + std_vec_new(&value, 1, 1); + txn_values_Entry_encode_into(&entry, &value); + txn_values_key(1, key); + if (!txn_values_sizes(key, sizeof(key), value.ptr, value.len, &sizes, &error)) + TXN_VALUES_TESTS_UNWRAP_FAILED(__LINE__, 9, &error); + std_vec_free_Checked(&value, 1, 1); + return sizes; +} + +/* txn_values_tests_sized_is asserts a value's entries, rows, and heap bytes. */ +static void +txn_values_tests_sized_is(uint32 line, uint64 stored, uint64 entries, uint64 rows, uint64 heap_bytes) +{ + TxnBaseSizes sizes = txn_values_tests_sized(stored); + + STD_ASSERT(line, 5, sizes.entries == entries && sizes.rows == rows && sizes.heap_bytes == heap_bytes, + "a value counts heap's toast chunks"); +} + +/* + * test_txn_values_tests_a_value_counts_heaps_toast_chunks: a value counts the + * toast chunks heap stores it in, TOAST_MAX_CHUNK_SIZE bytes each but its + * last, and their heap tuples' bytes with their line pointers, a short last + * chunk taking a one-byte varlena header. + */ +void +test_txn_values_tests_a_value_counts_heaps_toast_chunks(void) +{ + STD_ASSERT_EQ_U64(__LINE__, 5, TXN_VALUES_TOAST_MAX_CHUNK_SIZE, 1996); + txn_values_tests_sized_is(__LINE__, 6400, 1, 4, 3 * 2036 + 452); + txn_values_tests_sized_is(__LINE__, 2560, 1, 2, 2036 + 604); + txn_values_tests_sized_is(__LINE__, 2112, 1, 2, 2036 + 156); + txn_values_tests_sized_is(__LINE__, 1996, 1, 1, 2036); +} + +/* + * test_txn_values_tests_a_take_down_takes_one_from_a_count_and_removes_the_last: + * a take-down takes its amount from a value's count and removes the entry at + * its last, an add then a take-down leaves the count as it was, and a key + * with no entry, one that doesn't decode, or a count below the amount is + * refused. + */ +void +test_txn_values_tests_a_take_down_takes_one_from_a_count_and_removes_the_last(void) +{ + TxnValuesEntry entry = {.address = chunks_address_Address_of((const uint8 *) "blob", 4), + .locator = chunks_trailer_Locator_Default_default(), .raw = 4000, .stored = 4000, .count = 2}; + TxnValuesEntry decoded; + StdVec two; + StdVec one; + StdVec three; + StdVec back; + StdVec scratch; + HostError error; + + std_vec_new(&two, 1, 1); + std_vec_new(&one, 1, 1); + std_vec_new(&three, 1, 1); + std_vec_new(&back, 1, 1); + std_vec_new(&scratch, 1, 1); + txn_values_Entry_encode_into(&entry, &two); + STD_ASSERT(__LINE__, 5, txn_values_taken_down(true, two.ptr, two.len, &one, 1) == TXN_MEMTABLE_MAP_STEP_PUT, + "a take-down of one from two puts"); + if (!txn_values_Entry_decode(one.ptr, one.len, &decoded, &error)) + TXN_VALUES_TESTS_UNWRAP_FAILED(__LINE__, 16, &error); + STD_ASSERT_EQ_U64(__LINE__, 5, decoded.count, 1); + STD_ASSERT(__LINE__, 5, + txn_values_taken_down(true, one.ptr, one.len, &scratch, 1) == TXN_MEMTABLE_MAP_STEP_REMOVE, + "a take-down of the last removes the entry"); + STD_ASSERT(__LINE__, 5, + txn_values_taken_down(true, two.ptr, two.len, &scratch, 2) == TXN_MEMTABLE_MAP_STEP_REMOVE, + "a take-down of the whole count removes the entry"); + STD_ASSERT(__LINE__, 5, + txn_values_taken_down(true, two.ptr, two.len, &scratch, 3) == TXN_MEMTABLE_MAP_STEP_REFUSE, + "a take-down past the count is refused"); + STD_ASSERT(__LINE__, 5, txn_values_added(true, two.ptr, two.len, &three) == TXN_MEMTABLE_MAP_STEP_PUT, + "an add puts"); + STD_ASSERT(__LINE__, 5, txn_values_taken_down(true, three.ptr, three.len, &back, 1) == TXN_MEMTABLE_MAP_STEP_PUT, + "a take-down after an add puts"); + STD_ASSERT(__LINE__, 5, back.len == two.len && memcmp(back.ptr, two.ptr, two.len) == 0, + "an add then a take-down leaves the count as it was"); + STD_ASSERT(__LINE__, 5, txn_values_taken_down(false, NULL, 0, &scratch, 1) == TXN_MEMTABLE_MAP_STEP_REFUSE, + "a key with no entry is refused"); + STD_ASSERT(__LINE__, 5, txn_values_taken_down(true, two.ptr, 10, &scratch, 1) == TXN_MEMTABLE_MAP_STEP_REFUSE, + "an entry that doesn't decode is refused"); + std_vec_free_Checked(&scratch, 1, 1); + std_vec_free_Checked(&back, 1, 1); + std_vec_free_Checked(&three, 1, 1); + std_vec_free_Checked(&one, 1, 1); + std_vec_free_Checked(&two, 1, 1); +} + +#endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/visible.c b/src/doltgres/txn/c/visible.c index f7a42a6479bb5..4e929daf8733f 100644 --- a/src/doltgres/txn/c/visible.c +++ b/src/doltgres/txn/c/visible.c @@ -39,10 +39,13 @@ #include "doltgres/txn/visible.h" #include "port/pg_bswap.h" #ifdef DOLTGRES_TEST_TXN +#include "doltgres/repo/lib.h" #include "doltgres/simc/simc.h" #include "doltgres/std/test.h" +#include "doltgres/txn/memtable/slice.h" #include "doltgres/txn/memtable/table.h" #include "doltgres/txn/memtable/testing.h" +#include "doltgres/txn/state.h" #endif #define RUST_FILE "txn" STD_SEP "src" STD_SEP "visible.rs" @@ -1853,9 +1856,10 @@ test_txn_visible_tests_a_running_writers_version_leaves_its_ran_answer_as_it_was HostError error; bool has = true; bool ran = true; + TxnStatusRoomed unroomed = {0}; if (!txn_memtable_table_alone(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, host_shared_Held_record(&held), - (const uint8 *) "k", 1, &change, &handle, &error) || + (const uint8 *) "k", 1, &change, false, unroomed, &handle, &error) || !host_delta_hold(io, &hold, &error)) STD_RESULT_EXPECT_FAILED(__LINE__, 5, "called `Result::unwrap()` on an `Err` value", &error, host_Error_Debug_fmt); @@ -1873,4 +1877,2348 @@ test_txn_visible_tests_a_running_writers_version_leaves_its_ran_answer_as_it_was simc_TestIo_drop(&test_io); } +/* txn_visible_tests_made returns a new storage's state, by its database, in a repository made for it. */ +static HostSharedHeld +txn_visible_tests_made(HostIoIo *io, uint32 database) +{ + HostStorage storage = {.tablespace = HOST_PG_DEFAULT, .database = database, .relation = 16384}; + RepoRepo repo; + HostSharedReader reading; + HostSharedHeld held; + ChunksErrorError failed; + HostError error; + + if (!repo_Repo_create(io, database, HOST_PG_DEFAULT, &repo, &failed)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &failed, + chunks_error_Error_Debug_fmt); + if (!host_shared_read(io, &reading, &error) || + !txn_state_state(io, &repo, &reading, &storage, TXN_STATE_PERSISTENCE_PERMANENT, false, NULL, &held, + &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + host_shared_Reader_drop(&reading); + repo_Repo_drop(&repo); + return held; +} + +/* txn_visible_tests_by returns a writer of an xid under a top-level one, 0 for none, at a command. */ +static TxnMemtableVersionWriter +txn_visible_tests_by(HostXid xid, uint64 top, uint32 command) +{ + TxnMemtableVersionWriter writer = {.xid = xid.f0, .has_top = top != 0, .top = top, .command = command}; + + return writer; +} + +/* txn_visible_tests_pushed pushes a version of a key by a writer doing an op. */ +static void +txn_visible_tests_pushed(HostIoIo *io, HostSharedRecord state, const char *key, TxnMemtableVersionWriter writer, + const TxnMemtableOp *op) +{ + TxnMemtableVersionChange change = {.writer = writer, .op = op, .lock = TXN_MEMTABLE_LOCK_MODE_UPDATE, + .shown = {.tag = TXN_MEMTABLE_VERSION_SHOWN_COMMAND}}; + TxnMemtableVersionHandle handle; + HostError error; + + if (!txn_memtable_table_push(io, state, (const uint8 *) key, strlen(key), &change, &handle, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); +} + +/* txn_visible_tests_written pushes a row of a key by a writer with a tuple id, or with 0 the row's delete. */ +static void +txn_visible_tests_written(HostIoIo *io, HostSharedRecord state, const char *key, TxnMemtableVersionWriter writer, + uint64 tuple) +{ + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_DELETE}; + + if (tuple != 0) + { + op = txn_memtable_testing_put((const uint8 *) "row", 3); + op.u.put.tuple.f0 = tuple; + } + txn_visible_tests_pushed(io, state, key, writer, &op); + txn_memtable_Op_drop(&op); +} + +/* txn_visible_tests_sees returns the tuple ids of the row versions of a key that a snapshot sees, newest first, in a Vec. */ +static StdVec +txn_visible_tests_sees(HostIoIo *io, HostSharedRecord state, const char *key, const HostSnapshot *snapshot) +{ + TxnVisibleRule rule = {&TXN_VISIBLE_SNAPSHOT_RULE_OPS, snapshot}; + HostDeltaHold hold; + StdVec versions; + TxnVisibleSeen seen; + StdVec tuples; + HostError error; + + if (!host_delta_hold(io, &hold, &error) || + !txn_memtable_table_versions(io, &hold, state, (const uint8 *) key, strlen(key), &versions, &error) || + !txn_visible_seen(io, rule, versions.ptr, versions.len, NULL, &seen, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + std_vec_new(&tuples, sizeof(uint64), _Alignof(uint64)); + for (size_t k = 0; k < seen.rows.len; k++) + std_vec_push_Checked(&tuples, &STD_VEC_AT(&seen.rows, TxnVisibleSeenRow, k).row.put.tuple.f0, sizeof(uint64), + _Alignof(uint64)); + txn_visible_Seen_drop(&seen); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + return tuples; +} + +/* txn_visible_tests_sees_is asserts at its line that a snapshot sees exactly the tuple ids given of a key, newest first. */ +static void +txn_visible_tests_sees_is(uint32 line, HostIoIo *io, HostSharedRecord state, const char *key, + const HostSnapshot *snapshot, const uint64 *wanted, size_t wanted_len) +{ + StdVec seen = txn_visible_tests_sees(io, state, key, snapshot); + bool same = seen.len == wanted_len; + + for (size_t k = 0; same && k < wanted_len; k++) + same = STD_VEC_AT(&seen, uint64, k) == wanted[k]; + STD_ASSERT(line, 0, same, "a snapshot sees the key's row versions it should"); + std_vec_free_Checked(&seen, sizeof(uint64), _Alignof(uint64)); +} + +/* + * S1's case (PRevPF 5's prevpf-258): a snapshot taken while X runs, after X's + * subtransaction wrote a row, reads the subtransaction's row invisible once X + * commits, whether the snapshot's subtransactions overflowed or not, and a + * snapshot taken after the commit sees it. + */ +void +test_txn_visible_tests_a_subtransactions_row_stays_invisible_to_a_snapshot_taken_while_its_transaction_ran(void) +{ + static const char *const keys[2] = {"few", "many"}; + static const uint32 counts[2] = {1, 70}; + static const uint64 one[1] = {1}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9340); + + for (size_t k = 0; k < lengthof(keys); k++) + { + HostXid top = simc_begin(); + HostXid first = simc_begin_under(top); + SimcTaken taken; + SimcTaken later; + HostSnapshot lent; + HostSnapshot after; + + for (uint32 sub = 1; sub < counts[k]; sub++) + simc_begin_under(top); + txn_visible_tests_written(io, host_shared_Held_record(&held), keys[k], txn_visible_tests_by(first, top.f0, 0), + 1); + taken = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + lent = simc_Taken_lent(&taken); + STD_ASSERT(__LINE__, 0, lent.overflowed == (counts[k] > 64), "taken.lent().overflowed == (subs > 64)"); + simc_commit(top); + txn_visible_tests_sees_is(__LINE__, io, host_shared_Held_record(&held), keys[k], &lent, NULL, 0); + later = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + after = simc_Taken_lent(&later); + txn_visible_tests_sees_is(__LINE__, io, host_shared_Held_record(&held), keys[k], &after, one, 1); + simc_Taken_drop(&later); + simc_Taken_drop(&taken); + } + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* + * Heap's MVCC rule over a delete: a committed row stays visible to a snapshot + * taken while its deleter ran, after the deleter commits, and a later + * snapshot doesn't see the row; a recovery-shaped snapshot finds the running + * deleter in subxip. An aborted update between the row and its delete is + * passed over: the delete ends the row, and the update's row is seen by no + * snapshot. + */ +void +test_txn_visible_tests_a_rows_delete_is_seen_only_by_a_snapshot_taken_after_its_commit(void) +{ + static const uint64 one[1] = {1}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9341); + HostXid writer = simc_begin(); + HostXid updater = simc_begin(); + HostXid deleter; + SimcTaken before; + SimcTaken recovery; + SimcTaken later; + HostSnapshot lent; + + txn_visible_tests_written(io, host_shared_Held_record(&held), "k", txn_visible_tests_by(writer, 0, 0), 1); + simc_commit(writer); + txn_visible_tests_written(io, host_shared_Held_record(&held), "k", txn_visible_tests_by(updater, 0, 0), 2); + simc_abort(updater); + deleter = simc_begin(); + txn_visible_tests_written(io, host_shared_Held_record(&held), "k", txn_visible_tests_by(deleter, 0, 0), 0); + before = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + recovery = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, true); + lent = simc_Taken_lent(&recovery); + txn_visible_tests_sees_is(__LINE__, io, host_shared_Held_record(&held), "k", &lent, one, 1); + simc_commit(deleter); + lent = simc_Taken_lent(&before); + txn_visible_tests_sees_is(__LINE__, io, host_shared_Held_record(&held), "k", &lent, one, 1); + later = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + lent = simc_Taken_lent(&later); + txn_visible_tests_sees_is(__LINE__, io, host_shared_Held_record(&held), "k", &lent, NULL, 0); + simc_Taken_drop(&later); + simc_Taken_drop(&recovery); + simc_Taken_drop(&before); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* TxnVisibleTestsFound is a row version a read found: its tuple id and what a Dirty read found running. */ +typedef struct TxnVisibleTestsFound +{ + uint64 tuple; + TxnVisibleDirty dirty; +} TxnVisibleTestsFound; + +/* TxnVisibleTestsRead is read's answer: the row versions a snapshot sees, newest first, and the base's rows it keeps. */ +typedef struct TxnVisibleTestsRead +{ + /* rows is a Vec. */ + StdVec rows; + /* kept is a Vec. */ + StdVec kept; +} TxnVisibleTestsRead; + +/* TxnVisibleTestsChanged is a row's tuple id and its changer's xid, if it has one. */ +typedef struct TxnVisibleTestsChanged +{ + uint64 tuple; + bool has_changer; + HostXid changer; +} TxnVisibleTestsChanged; + +/* TxnVisibleTestsConflict is a conflict a serializable read found: its row's tuple id and its writer's xid. */ +typedef struct TxnVisibleTestsConflict +{ + uint64 tuple; + uint64 xid; +} TxnVisibleTestsConflict; + +/* txn_visible_tests_u64s_Debug_fmt is a Vec's Debug. */ +static bool +txn_visible_tests_u64s_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *u64s = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < u64s->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(u64s, uint64, k), std_u64_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_visible_tests_found_Debug_fmt is a found row's Debug, its tuple id and its Dirty. */ +static bool +txn_visible_tests_found_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnVisibleTestsFound *found = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&found->tuple, std_u64_Debug_fmt), + STD_ITEM(&found->dirty, txn_visible_Dirty_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_visible_tests_founds_Debug_fmt is a Vec of found rows' Debug. */ +static bool +txn_visible_tests_founds_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *founds = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < founds->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(founds, TxnVisibleTestsFound, k), txn_visible_tests_found_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_visible_tests_read_Debug_fmt is a read's Debug, its rows and its kept base rows. */ +static bool +txn_visible_tests_read_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnVisibleTestsRead *read = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&read->rows, txn_visible_tests_founds_Debug_fmt), + STD_ITEM(&read->kept, txn_visible_tests_u64s_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_visible_tests_changer_Debug_fmt is an Option's Debug, a changer's xid. */ +static bool +txn_visible_tests_changer_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnVisibleTestsChanged *changed = value; + StdOptionRef changer = {changed->has_changer, STD_ITEM(&changed->changer, host_Xid_Debug_fmt)}; + + return std_option_Debug_fmt(&changer, f); +} + +/* txn_visible_tests_changed_Debug_fmt is a changed row's Debug, its tuple id and its changer's xid. */ +static bool +txn_visible_tests_changed_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnVisibleTestsChanged *changed = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&changed->tuple, std_u64_Debug_fmt), + STD_ITEM(changed, txn_visible_tests_changer_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_visible_tests_changeds_Debug_fmt is a Vec of changed rows' Debug. */ +static bool +txn_visible_tests_changeds_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *changeds = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < changeds->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(changeds, TxnVisibleTestsChanged, k), + txn_visible_tests_changed_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_visible_tests_changers_Debug_fmt is a Vec of changers' Debug, each an Option. */ +static bool +txn_visible_tests_changers_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *changers = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < changers->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(changers, TxnVisibleTestsChanged, k), + txn_visible_tests_changer_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_visible_tests_conflict_Debug_fmt is a conflict's Debug, its tuple id and its writer's xid. */ +static bool +txn_visible_tests_conflict_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnVisibleTestsConflict *conflict = value; + StdTupleRef tuple = STD_TUPLE(2, STD_ITEM(&conflict->tuple, std_u64_Debug_fmt), + STD_ITEM(&conflict->xid, std_u64_Debug_fmt)); + + return std_tuple_Debug_fmt(&tuple, f); +} + +/* txn_visible_tests_conflicts_Debug_fmt is a Vec of conflicts' Debug. */ +static bool +txn_visible_tests_conflicts_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *conflicts = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < conflicts->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(conflicts, TxnVisibleTestsConflict, k), + txn_visible_tests_conflict_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_visible_tests_key_failed is an assert_eq!'s failure naming a key's bytes, as its message `{key:?}` shows them. */ +static void +txn_visible_tests_key_failed(const char *key, const void *left, StdFmtFn left_fmt, const void *right, + StdFmtFn right_fmt, uint32 line) +{ + StdBytes bytes = {(const uint8 *) key, strlen(key)}; + StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&bytes, std_bytes_Debug_fmt}}; + StdFmtArguments arguments = {pieces, 1, args}; + + std_assert_failed(RUST_FILE, line, 13, "==", left, left_fmt, right, right_fmt, &arguments); +} + +/* txn_visible_tests_put returns a row's put with a tuple id. */ +static TxnMemtablePut +txn_visible_tests_put(uint64 tuple) +{ + TxnMemtableOp op = txn_memtable_testing_put((const uint8 *) "row", 3); + + op.u.put.tuple.f0 = tuple; + return op.u.put; +} + +/* txn_visible_tests_pushing pushes a version of a key by a writer doing an op, taking the op. */ +static void +txn_visible_tests_pushing(HostIoIo *io, HostSharedRecord state, const char *key, TxnMemtableVersionWriter writer, + TxnMemtableOp op) +{ + txn_visible_tests_pushed(io, state, key, writer, &op); + txn_memtable_Op_drop(&op); +} + +/* txn_visible_tests_versioned holds a key's versions under a hold of the delta, as table's versions reads them. */ +static StdVec +txn_visible_tests_versioned(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, const char *key) +{ + StdVec versions; + HostError error; + + if (!txn_memtable_table_versions(io, hold, state, (const uint8 *) key, strlen(key), &versions, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return versions; +} + +/* txn_visible_tests_hold returns a hold of the delta. */ +static HostDeltaHold +txn_visible_tests_hold(HostIoIo *io) +{ + HostDeltaHold hold; + HostError error; + + if (!host_delta_hold(io, &hold, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return hold; +} + +/* txn_visible_tests_seen_of returns what a snapshot sees of versions, under a serializable read's bounds or none. */ +static TxnVisibleSeen +txn_visible_tests_seen_of(HostIoIo *io, const HostSnapshot *snapshot, StdVec *versions, const TxnVisibleSerial *serial) +{ + TxnVisibleRule rule = {&TXN_VISIBLE_SNAPSHOT_RULE_OPS, snapshot}; + TxnVisibleSeen seen; + HostError error; + + if (!txn_visible_seen(io, rule, versions->ptr, versions->len, serial, &seen, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return seen; +} + +/* txn_visible_tests_xid returns the xid of a version's writer, or none. */ +static TxnVisibleTestsChanged +txn_visible_tests_xid(uint64 tuple, const TxnMemtableVersionVersion *changer) +{ + TxnVisibleTestsChanged changed = {.tuple = tuple, .has_changer = changer != NULL}; + + if (changer != NULL) + { + TxnMemtableVersionRecord record = txn_memtable_version_Version_record(changer); + + changed.changer.f0 = txn_memtable_version_Record_writer(&record).xid; + } + return changed; +} + +/* + * txn_visible_tests_read returns what a snapshot sees of a key: the tuple ids + * of the row versions it sees, newest first, each with what a Dirty read + * found running, and of the base's rows with the given tuple ids. + */ +static TxnVisibleTestsRead +txn_visible_tests_read(HostIoIo *io, HostSharedRecord state, const char *key, const HostSnapshot *snapshot, + const uint64 *base, size_t base_len) +{ + TxnVisibleRule rule = {&TXN_VISIBLE_SNAPSHOT_RULE_OPS, snapshot}; + HostDeltaHold hold = txn_visible_tests_hold(io); + StdVec versions = txn_visible_tests_versioned(io, &hold, state, key); + TxnVisibleSeen seen = txn_visible_tests_seen_of(io, snapshot, &versions, NULL); + TxnVisibleTestsRead read; + + std_vec_new(&read.rows, sizeof(TxnVisibleTestsFound), _Alignof(TxnVisibleTestsFound)); + std_vec_new(&read.kept, sizeof(uint64), _Alignof(uint64)); + for (size_t k = 0; k < seen.rows.len; k++) + { + const TxnVisibleSeenRow *row = &STD_VEC_AT(&seen.rows, TxnVisibleSeenRow, k); + TxnVisibleTestsFound found = {row->row.put.tuple.f0, row->dirty}; + + std_vec_push_Checked(&read.rows, &found, sizeof(found), _Alignof(TxnVisibleTestsFound)); + } + for (size_t k = 0; k < base_len; k++) + { + bool kept; + TxnVisibleDirty dirty; + TxnMemtableVersionVersion *changer; + TxnMemtableVersionVersion *commander; + HostRow row = {base[k]}; + HostError error; + + if (!txn_visible_Seen_base(&seen, io, rule, row, &kept, &dirty, &changer, &commander, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + if (kept) + std_vec_push_Checked(&read.kept, &base[k], sizeof(uint64), _Alignof(uint64)); + } + txn_visible_Seen_drop(&seen); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + return read; +} + +/* txn_visible_tests_read_is asserts at a line that a read found exactly the rows and kept base rows given, freeing it. */ +static void +txn_visible_tests_read_is(uint32 line, TxnVisibleTestsRead *read, const TxnVisibleTestsFound *rows, size_t rows_len, + const uint64 *kept, size_t kept_len) +{ + TxnVisibleTestsRead wanted = {{(void *) rows, rows_len, rows_len}, {(void *) kept, kept_len, kept_len}}; + bool same = read->rows.len == rows_len && read->kept.len == kept_len; + + for (size_t k = 0; same && k < rows_len; k++) + same = STD_VEC_AT(&read->rows, TxnVisibleTestsFound, k).tuple == rows[k].tuple && + txn_visible_Dirty_PartialEq_eq(&STD_VEC_AT(&read->rows, TxnVisibleTestsFound, k).dirty, &rows[k].dirty); + for (size_t k = 0; same && k < kept_len; k++) + same = STD_VEC_AT(&read->kept, uint64, k) == kept[k]; + STD_ASSERT_EQ(line, 9, same, read, txn_visible_tests_read_Debug_fmt, &wanted, txn_visible_tests_read_Debug_fmt); + std_vec_free_Checked(&read->rows, sizeof(TxnVisibleTestsFound), _Alignof(TxnVisibleTestsFound)); + std_vec_free_Checked(&read->kept, sizeof(uint64), _Alignof(uint64)); +} + +/* txn_visible_tests_dirty_is asserts at a line that a Dirty read of a key finds exactly the rows given. */ +static void +txn_visible_tests_dirty_is(uint32 line, HostIoIo *io, HostSharedRecord state, const char *key, + const TxnVisibleTestsFound *rows, size_t rows_len) +{ + SimcTaken taken = simc_snapshot(HOST_SNAPSHOT_KIND_DIRTY, false); + HostSnapshot lent = simc_Taken_lent(&taken); + TxnVisibleTestsRead read = txn_visible_tests_read(io, state, key, &lent, NULL, 0); + + txn_visible_tests_read_is(line, &read, rows, rows_len, NULL, 0); + simc_Taken_drop(&taken); +} + +/* txn_visible_tests_taken_sees_is asserts at a line that a taken snapshot sees exactly the tuple ids given, freeing it. */ +static void +txn_visible_tests_taken_sees_is(uint32 line, HostIoIo *io, HostSharedRecord state, const char *key, SimcTaken taken, + const uint64 *wanted, size_t wanted_len) +{ + HostSnapshot lent = simc_Taken_lent(&taken); + + txn_visible_tests_sees_is(line, io, state, key, &lent, wanted, wanted_len); + simc_Taken_drop(&taken); +} + +/* txn_visible_tests_kind_seen_is asserts at a line whether a snapshot of a kind sees a row of a key, naming the key. */ +static void +txn_visible_tests_kind_seen_is(uint32 line, HostIoIo *io, HostSharedRecord state, const char *key, + HostSnapshotKind kind, bool wanted) +{ + SimcTaken taken = simc_snapshot(kind, false); + HostSnapshot lent = simc_Taken_lent(&taken); + StdVec seen = txn_visible_tests_sees(io, state, key, &lent); + bool got = seen.len != 0; + + if (got != wanted) + txn_visible_tests_key_failed(key, &got, std_bool_Debug_fmt, &wanted, std_bool_Debug_fmt, line); + std_vec_free_Checked(&seen, sizeof(uint64), _Alignof(uint64)); + simc_Taken_drop(&taken); +} + +/* + * test_txn_visible_tests_each_snapshot_kind_sees_the_versions_heaps_rule_sees: + * the other kinds' rules: Any sees every row version; Toast every one but an + * aborted writer's, an invalid one's among them; Self a running writer's row + * not, and a committed one's; Dirty a running writer's row, with that writer + * reported; a row FrozenTransactionId wrote is seen by MVCC; and + * NonVacuumable sees a committed delete's victim while its horizon is at the + * deleter, and not once the horizon passes the deleter. + */ +void +test_txn_visible_tests_each_snapshot_kind_sees_the_versions_heaps_rule_sees(void) +{ + static const char *const keys[] = {"run", "abort", "invalid"}; + static const bool any[] = {true, true, true}; + static const bool toast[] = {true, false, false}; + static const bool current[] = {false, false, false}; + static const uint64 one[] = {1}; + static const uint64 four[] = {4}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9342); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid running = simc_begin(); + HostXid aborted = simc_begin(); + HostXid deleter; + HostXid invalid = {0}; + HostXid frozen = {2}; + HostXid past; + TxnVisibleTestsFound found = {1, txn_visible_Dirty_Default_default()}; + + simc_abort(aborted); + txn_visible_tests_written(io, state, "run", txn_visible_tests_by(running, 0, 0), 1); + txn_visible_tests_written(io, state, "abort", txn_visible_tests_by(aborted, 0, 0), 2); + txn_visible_tests_written(io, state, "invalid", txn_visible_tests_by(invalid, 0, 0), 3); + txn_visible_tests_written(io, state, "frozen", txn_visible_tests_by(frozen, 0, 0), 4); + for (size_t k = 0; k < lengthof(keys); k++) + { + txn_visible_tests_kind_seen_is(__LINE__, io, state, keys[k], HOST_SNAPSHOT_KIND_ANY, any[k]); + txn_visible_tests_kind_seen_is(__LINE__, io, state, keys[k], HOST_SNAPSHOT_KIND_TOAST, toast[k]); + txn_visible_tests_kind_seen_is(__LINE__, io, state, keys[k], HOST_SNAPSHOT_KIND_CURRENT, current[k]); + } + found.dirty.has_xmin = true; + found.dirty.xmin = running; + txn_visible_tests_dirty_is(__LINE__, io, state, "run", &found, 1); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "frozen", simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false), four, + 1); + deleter = simc_begin(); + txn_visible_tests_written(io, state, "frozen", txn_visible_tests_by(deleter, 0, 0), 0); + simc_commit(deleter); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "frozen", simc_non_vacuumable(deleter), four, 1); + past.f0 = deleter.f0 + 1; + txn_visible_tests_taken_sees_is(__LINE__, io, state, "frozen", simc_non_vacuumable(past), NULL, 0); + simc_commit(running); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "run", simc_snapshot(HOST_SNAPSHOT_KIND_CURRENT, false), one, + 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_visible_tests_a_key_read_gives_every_row_version_the_snapshots_rule_sees: + * a key's read gives every row version heap's rule sees, as a heap scan + * returns each such tuple: under Dirty a running update's new row, its writer + * reported, and its old row, its ender reported, where MVCC and Self see the + * old row alone; a row one running transaction inserted and deleted is seen + * under Dirty with its writer reported and its ender not, since heap's rule + * returns at a running writer; once the update commits, Dirty and MVCC the + * new row alone, NonVacuumable the old row beside it while its horizon is at + * the updater, and Any both. + */ +void +test_txn_visible_tests_a_key_read_gives_every_row_version_the_snapshots_rule_sees(void) +{ + static const uint64 one[] = {1}; + static const uint64 two[] = {2}; + static const uint64 both[] = {2, 1}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9344); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid writer = simc_begin(); + HostXid updater; + HostXid past; + TxnVisibleTestsFound running[2] = {{2, txn_visible_Dirty_Default_default()}, + {1, txn_visible_Dirty_Default_default()}}; + TxnVisibleTestsFound gone; + TxnVisibleTestsFound committed = {2, txn_visible_Dirty_Default_default()}; + + txn_visible_tests_written(io, state, "k", txn_visible_tests_by(writer, 0, 0), 1); + simc_commit(writer); + updater = simc_begin(); + txn_visible_tests_written(io, state, "k", txn_visible_tests_by(updater, 0, 0), 2); + running[0].dirty.has_xmin = true; + running[0].dirty.xmin = updater; + running[1].dirty.has_xmax = true; + running[1].dirty.xmax = updater; + txn_visible_tests_dirty_is(__LINE__, io, state, "k", running, lengthof(running)); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "k", simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false), one, 1); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "k", simc_snapshot(HOST_SNAPSHOT_KIND_CURRENT, false), one, + 1); + txn_visible_tests_written(io, state, "gone", txn_visible_tests_by(updater, 0, 0), 3); + txn_visible_tests_written(io, state, "gone", txn_visible_tests_by(updater, 0, 1), 0); + gone = (TxnVisibleTestsFound) + { + 3, running[0].dirty + }; + txn_visible_tests_dirty_is(__LINE__, io, state, "gone", &gone, 1); + simc_commit(updater); + txn_visible_tests_dirty_is(__LINE__, io, state, "k", &committed, 1); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "k", simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false), two, 1); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "k", simc_non_vacuumable(updater), both, 2); + past.f0 = updater.f0 + 1; + txn_visible_tests_taken_sees_is(__LINE__, io, state, "k", simc_non_vacuumable(past), two, 1); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "k", simc_snapshot(HOST_SNAPSHOT_KIND_ANY, false), both, 2); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_visible_tests_a_row_named_by_its_tuple_id_is_ended_by_the_version_that_names_it: + * the keys that hold more than one row. A counted key's copy is ended by the + * removal that names its tuple id, in the delta and in the base, and by no + * other; a row named by its tuple id is replaced by the update that names it, + * beside its key's other row; a removal whose writer still runs has ended + * nothing for an MVCC snapshot; and a version that sets the key's value ends + * every older row of the key and the base's. + */ +void +test_txn_visible_tests_a_row_named_by_its_tuple_id_is_ended_by_the_version_that_names_it(void) +{ + static const uint64 counted_base[] = {10, 11, 12}; + static const uint64 rows_base[] = {10, 11}; + static const uint64 counted_kept[] = {11, 12}; + static const uint64 rows_kept[] = {11}; + static const uint64 after_kept[] = {12}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9345); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid writer = simc_begin(); + HostXid remover; + HostXid setter; + TxnMemtableVersionWriter by; + TxnMemtableOp op; + SimcTaken mvcc; + SimcTaken after; + HostSnapshot lent; + TxnVisibleTestsRead read; + TxnVisibleTestsFound counted[] = {{2, txn_visible_Dirty_Default_default()}}; + TxnVisibleTestsFound rows[] = {{3, txn_visible_Dirty_Default_default()}, {2, txn_visible_Dirty_Default_default()}}; + TxnVisibleTestsFound set[] = {{4, txn_visible_Dirty_Default_default()}}; + + by = txn_visible_tests_by(writer, 0, 0); + for (uint64 tuple = 1; tuple <= 2; tuple++) + { + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_ADD, .u.add = txn_visible_tests_put(tuple) + }; + txn_visible_tests_pushing(io, state, "counted", by, op); + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_ADD_ROW, .u.add_row = txn_visible_tests_put(tuple) + }; + txn_visible_tests_pushing(io, state, "rows", by, op); + } + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_REMOVE, .u.remove = + { + { + 1 + } + } + }; + txn_visible_tests_pushing(io, state, "counted", by, op); + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_REMOVE, .u.remove = + { + { + 10 + } + } + }; + txn_visible_tests_pushing(io, state, "counted", by, op); + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_UPDATE_ROW, .u.update_row = + { + { + 1 + }, txn_visible_tests_put(3) + } + }; + txn_visible_tests_pushing(io, state, "rows", by, op); + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_DELETE_ROW, .u.delete_row = + { + { + 10 + } + } + }; + txn_visible_tests_pushing(io, state, "rows", by, op); + simc_commit(writer); + remover = simc_begin(); + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_REMOVE, .u.remove = + { + { + 11 + } + } + }; + txn_visible_tests_pushing(io, state, "counted", txn_visible_tests_by(remover, 0, 0), op); + mvcc = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + lent = simc_Taken_lent(&mvcc); + read = txn_visible_tests_read(io, state, "counted", &lent, counted_base, lengthof(counted_base)); + txn_visible_tests_read_is(__LINE__, &read, counted, lengthof(counted), counted_kept, lengthof(counted_kept)); + read = txn_visible_tests_read(io, state, "rows", &lent, rows_base, lengthof(rows_base)); + txn_visible_tests_read_is(__LINE__, &read, rows, lengthof(rows), rows_kept, lengthof(rows_kept)); + simc_commit(remover); + setter = simc_begin(); + txn_visible_tests_written(io, state, "rows", txn_visible_tests_by(setter, 0, 0), 4); + simc_commit(setter); + after = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + lent = simc_Taken_lent(&after); + read = txn_visible_tests_read(io, state, "rows", &lent, rows_base, lengthof(rows_base)); + txn_visible_tests_read_is(__LINE__, &read, set, lengthof(set), NULL, 0); + read = txn_visible_tests_read(io, state, "counted", &lent, counted_base, lengthof(counted_base)); + txn_visible_tests_read_is(__LINE__, &read, counted, lengthof(counted), after_kept, lengthof(after_kept)); + simc_Taken_drop(&after); + simc_Taken_drop(&mvcc); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* TxnVisibleTestsShown is a stamp case's read: each row's changer, and each named base row's. */ +typedef struct TxnVisibleTestsShown +{ + /* rows is a Vec. */ + StdVec rows; + /* based is a Vec, whose tuple ids are the base's. */ + StdVec based; +} TxnVisibleTestsShown; + +/* txn_visible_tests_shown returns each row version an Any snapshot sees of a key with its changer, and each base row's. */ +static TxnVisibleTestsShown +txn_visible_tests_shown(HostIoIo *io, HostSharedRecord state, const char *key, const HostSnapshot *any, + const uint64 *base, size_t base_len) +{ + TxnVisibleRule rule = {&TXN_VISIBLE_SNAPSHOT_RULE_OPS, any}; + HostDeltaHold hold = txn_visible_tests_hold(io); + StdVec versions = txn_visible_tests_versioned(io, &hold, state, key); + TxnVisibleSeen seen = txn_visible_tests_seen_of(io, any, &versions, NULL); + TxnVisibleTestsShown shown; + + std_vec_new(&shown.rows, sizeof(TxnVisibleTestsChanged), _Alignof(TxnVisibleTestsChanged)); + std_vec_new(&shown.based, sizeof(TxnVisibleTestsChanged), _Alignof(TxnVisibleTestsChanged)); + for (size_t k = 0; k < seen.rows.len; k++) + { + const TxnVisibleSeenRow *row = &STD_VEC_AT(&seen.rows, TxnVisibleSeenRow, k); + TxnVisibleTestsChanged changed = txn_visible_tests_xid(row->row.put.tuple.f0, row->changer); + + std_vec_push_Checked(&shown.rows, &changed, sizeof(changed), _Alignof(TxnVisibleTestsChanged)); + } + for (size_t k = 0; k < base_len; k++) + { + bool kept; + TxnVisibleDirty dirty; + TxnMemtableVersionVersion *changer; + TxnMemtableVersionVersion *commander; + TxnVisibleTestsChanged changed; + HostRow row = {base[k]}; + HostError error; + + if (!txn_visible_Seen_base(&seen, io, rule, row, &kept, &dirty, &changer, &commander, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + changed = txn_visible_tests_xid(base[k], changer); + std_vec_push_Checked(&shown.based, &changed, sizeof(changed), _Alignof(TxnVisibleTestsChanged)); + } + txn_visible_Seen_drop(&seen); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + return shown; +} + +/* txn_visible_tests_changed_eq returns whether two Vecs of changed rows hold the same, comparing tuple ids only when rows is set. */ +static bool +txn_visible_tests_changed_eq(const StdVec *got, const TxnVisibleTestsChanged *wanted, size_t wanted_len, bool rows) +{ + bool same = got->len == wanted_len; + + for (size_t k = 0; same && k < wanted_len; k++) + { + const TxnVisibleTestsChanged *one = &STD_VEC_AT(got, TxnVisibleTestsChanged, k); + + same = (!rows || one->tuple == wanted[k].tuple) && one->has_changer == wanted[k].has_changer && + (!one->has_changer || one->changer.f0 == wanted[k].changer.f0); + } + return same; +} + +/* txn_visible_tests_shown_is asserts at a line that a key's rows show the changers given, freeing what it read. */ +static void +txn_visible_tests_shown_is(uint32 line, TxnVisibleTestsShown *shown, const TxnVisibleTestsChanged *wanted, + size_t wanted_len) +{ + StdVec want = {(void *) wanted, wanted_len, wanted_len}; + + STD_ASSERT_EQ(line, 9, txn_visible_tests_changed_eq(&shown->rows, wanted, wanted_len, true), &shown->rows, + txn_visible_tests_changeds_Debug_fmt, &want, txn_visible_tests_changeds_Debug_fmt); + std_vec_free_Checked(&shown->rows, sizeof(TxnVisibleTestsChanged), _Alignof(TxnVisibleTestsChanged)); + std_vec_free_Checked(&shown->based, sizeof(TxnVisibleTestsChanged), _Alignof(TxnVisibleTestsChanged)); +} + +/* txn_visible_tests_changed returns a row's tuple id with its changer's xid, an xid of zero for none. */ +static TxnVisibleTestsChanged +txn_visible_tests_changed(uint64 tuple, HostXid changer) +{ + TxnVisibleTestsChanged changed = {.tuple = tuple, .has_changer = changer.f0 != 0, .changer = changer}; + + return changed; +} + +/* txn_visible_tests_lock returns a lock of a key, or with a tuple id of a row it names, in update mode. */ +static TxnMemtableOp +txn_visible_tests_lock(uint64 row) +{ + TxnMemtableOp op = {.tag = TXN_MEMTABLE_OP_LOCK, .u.lock = TXN_MEMTABLE_LOCK_MODE_UPDATE}; + + if (row != 0) + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_LOCK_ROW, .u.lock_row = + { + { + row + }, TXN_MEMTABLE_LOCK_MODE_UPDATE + } + }; + return op; +} + +/* + * test_txn_visible_tests_a_rows_stamp_shows_the_last_change_above_it_whatever_its_outcome: + * decision 4's stamp (bugs row 60): a row's changer is its ender, or with + * none the newest version above it that sets or locks its key or names it, + * whatever its outcome, as heap's raw xmax is the last change that set it. A + * row with an aborted update above it shows the aborted writer; an aborted + * update and then a running lock, the locker; a committed lock and then a + * committed delete, the deleter, read under Any, which sees the deleted row; + * a row whose committed update's new row was then locked shows the updater, + * and the new row the locker; a row deleted and inserted again at its key + * shows the deleter; a row named by its tuple id shows the version that names + * it and not the one that names its key's other row; and a base row the + * same. + */ +void +test_txn_visible_tests_a_rows_stamp_shows_the_last_change_above_it_whatever_its_outcome(void) +{ + static const uint64 base[] = {20, 21}; + static const HostXid none = {0}; + const TxnMemtableOp unnamed = {.tag = TXN_MEMTABLE_OP_DELETE_ROW, .u.delete_row = {{20}}}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9346); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid writer = simc_begin(); + HostXid aborted = simc_begin(); + HostXid locker = simc_begin(); + HostXid deleter = simc_begin(); + SimcTaken any; + HostSnapshot lent; + TxnVisibleTestsShown shown; + TxnVisibleTestsChanged updated[2]; + TxnVisibleTestsChanged locked[2]; + TxnVisibleTestsChanged deleted[1]; + TxnVisibleTestsChanged relocked[2]; + TxnVisibleTestsChanged again[2]; + TxnVisibleTestsChanged rows[2]; + TxnVisibleTestsChanged based[2]; + StdVec want = {based, 2, 2}; + + txn_visible_tests_written(io, state, "updated", txn_visible_tests_by(writer, 0, 0), 1); + txn_visible_tests_written(io, state, "updated", txn_visible_tests_by(aborted, 0, 0), 2); + txn_visible_tests_written(io, state, "locked", txn_visible_tests_by(writer, 0, 0), 3); + txn_visible_tests_written(io, state, "locked", txn_visible_tests_by(aborted, 0, 0), 4); + txn_visible_tests_pushing(io, state, "locked", txn_visible_tests_by(locker, 0, 0), txn_visible_tests_lock(0)); + txn_visible_tests_written(io, state, "deleted", txn_visible_tests_by(writer, 0, 0), 5); + txn_visible_tests_pushing(io, state, "deleted", txn_visible_tests_by(writer, 0, 1), txn_visible_tests_lock(0)); + txn_visible_tests_written(io, state, "deleted", txn_visible_tests_by(deleter, 0, 0), 0); + for (uint64 tuple = 6; tuple <= 7; tuple++) + { + TxnMemtableOp added = {.tag = TXN_MEMTABLE_OP_ADD_ROW}; + + added.u.add_row = txn_visible_tests_put(tuple); + txn_visible_tests_pushing(io, state, "rows", txn_visible_tests_by(writer, 0, 0), added); + } + txn_visible_tests_written(io, state, "relocked", txn_visible_tests_by(writer, 0, 0), 8); + txn_visible_tests_written(io, state, "relocked", txn_visible_tests_by(deleter, 0, 0), 9); + txn_visible_tests_pushing(io, state, "relocked", txn_visible_tests_by(locker, 0, 0), txn_visible_tests_lock(0)); + txn_visible_tests_written(io, state, "again", txn_visible_tests_by(writer, 0, 0), 10); + txn_visible_tests_written(io, state, "again", txn_visible_tests_by(deleter, 0, 0), 0); + txn_visible_tests_written(io, state, "again", txn_visible_tests_by(locker, 0, 0), 11); + txn_visible_tests_pushing(io, state, "rows", txn_visible_tests_by(locker, 0, 0), txn_visible_tests_lock(6)); + txn_visible_tests_pushing(io, state, "rows", txn_visible_tests_by(aborted, 0, 0), unnamed); + simc_abort(aborted); + simc_commit(writer); + simc_commit(deleter); + any = simc_snapshot(HOST_SNAPSHOT_KIND_ANY, false); + lent = simc_Taken_lent(&any); + updated[0] = txn_visible_tests_changed(2, none); + updated[1] = txn_visible_tests_changed(1, aborted); + shown = txn_visible_tests_shown(io, state, "updated", &lent, NULL, 0); + txn_visible_tests_shown_is(__LINE__, &shown, updated, lengthof(updated)); + locked[0] = txn_visible_tests_changed(4, locker); + locked[1] = txn_visible_tests_changed(3, locker); + shown = txn_visible_tests_shown(io, state, "locked", &lent, NULL, 0); + txn_visible_tests_shown_is(__LINE__, &shown, locked, lengthof(locked)); + deleted[0] = txn_visible_tests_changed(5, deleter); + shown = txn_visible_tests_shown(io, state, "deleted", &lent, NULL, 0); + txn_visible_tests_shown_is(__LINE__, &shown, deleted, lengthof(deleted)); + relocked[0] = txn_visible_tests_changed(9, locker); + relocked[1] = txn_visible_tests_changed(8, deleter); + shown = txn_visible_tests_shown(io, state, "relocked", &lent, NULL, 0); + txn_visible_tests_shown_is(__LINE__, &shown, relocked, lengthof(relocked)); + again[0] = txn_visible_tests_changed(11, none); + again[1] = txn_visible_tests_changed(10, deleter); + shown = txn_visible_tests_shown(io, state, "again", &lent, NULL, 0); + txn_visible_tests_shown_is(__LINE__, &shown, again, lengthof(again)); + rows[0] = txn_visible_tests_changed(7, none); + rows[1] = txn_visible_tests_changed(6, locker); + based[0] = txn_visible_tests_changed(20, aborted); + based[1] = txn_visible_tests_changed(21, none); + shown = txn_visible_tests_shown(io, state, "rows", &lent, base, lengthof(base)); + STD_ASSERT_EQ(__LINE__, 9, txn_visible_tests_changed_eq(&shown.based, based, lengthof(based), false), + &shown.based, txn_visible_tests_changers_Debug_fmt, &want, txn_visible_tests_changers_Debug_fmt); + txn_visible_tests_shown_is(__LINE__, &shown, rows, lengthof(rows)); + simc_Taken_drop(&any); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* TxnVisibleTestsCase is a row of the historic case: its key, its writer and command, its deleter and command, and whether it is seen. */ +typedef struct TxnVisibleTestsCase +{ + const char *key; + int writer; + uint32 command; + /* deleter is the deleter's place among the case's xids, or -1 for none. */ + int deleter; + uint32 deleted; + bool seen; +} TxnVisibleTestsCase; + +/* txn_visible_tests_named_sees_is asserts at a line that a snapshot sees exactly the tuple ids given of a key, naming the key. */ +static void +txn_visible_tests_named_sees_is(uint32 line, HostIoIo *io, HostSharedRecord state, const char *key, + const HostSnapshot *snapshot, const uint64 *wanted, size_t wanted_len) +{ + StdVec seen = txn_visible_tests_sees(io, state, key, snapshot); + StdVec want = {(void *) wanted, wanted_len, wanted_len}; + bool same = seen.len == wanted_len; + + for (size_t k = 0; same && k < wanted_len; k++) + same = STD_VEC_AT(&seen, uint64, k) == wanted[k]; + if (!same) + { + StdStr name = {key, strlen(key)}; + StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&name, std_str_Display_fmt}}; + StdFmtArguments arguments = {pieces, 1, args}; + + std_assert_failed(RUST_FILE, line, 13, "==", &seen, txn_visible_tests_u64s_Debug_fmt, &want, + txn_visible_tests_u64s_Debug_fmt, &arguments); + } + std_vec_free_Checked(&seen, sizeof(uint64), _Alignof(uint64)); +} + +/* + * test_txn_visible_tests_a_historic_snapshot_sees_its_transactions_rows_by_their_commands: + * Historic MVCC's rule, heap's HeapTupleSatisfiesHistoricMVCC, under a + * snapshot whose bounds are two of the case's xids, whose xip holds one xid + * that committed between them, and whose subxip holds the decoded + * transaction. Another writer's row is seen when the writer committed below + * xmin or is in xip, and not when it aborted, isn't in xip, or is at xmax; a + * delete counts by the same rule. The decoded transaction's row is seen from + * the command after the one its stamp names, and not at that command; its + * delete hides a row from the command after the deleter's, and not before + * (CommPF 7's ruling (S), Differences row 39). + */ +void +test_txn_visible_tests_a_historic_snapshot_sees_its_transactions_rows_by_their_commands(void) +{ + enum + { + OLD, ABORTED, LISTED, UNLISTED, OWN, LATER, + }; + static const TxnVisibleTestsCase cases[] = { + {"old", OLD, 0, -1, 0, true}, + {"aborted", ABORTED, 0, -1, 0, false}, + {"listed", LISTED, 0, -1, 0, true}, + {"unlisted", UNLISTED, 0, -1, 0, false}, + {"later", LATER, 0, -1, 0, false}, + {"old deleted by old", OLD, 0, OLD, 1, false}, + {"old deleted by aborted", OLD, 0, ABORTED, 0, true}, + {"old deleted by listed", OLD, 0, LISTED, 0, false}, + {"old deleted by unlisted", OLD, 0, UNLISTED, 0, true}, + {"old deleted by later", OLD, 0, LATER, 0, true}, + {"own before", OWN, 4, -1, 0, true}, + {"own at the command", OWN, 5, -1, 0, false}, + {"old deleted by own before", OLD, 0, OWN, 4, false}, + {"old deleted by own at the command", OLD, 0, OWN, 5, true}, + {"own deleted by own", OWN, 2, OWN, 3, false}, + }; + static const uint64 fifteen[] = {15}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9343); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid xids[6]; + SimcTaken taken; + SimcTaken earlier; + HostSnapshot lent; + HostCommand at_five = {5}; + HostCommand at_three = {3}; + + for (size_t k = 0; k < lengthof(xids); k++) + xids[k] = simc_begin(); + simc_abort(xids[ABORTED]); + simc_commit(xids[OLD]); + simc_commit(xids[LISTED]); + simc_commit(xids[UNLISTED]); + simc_commit(xids[OWN]); + simc_commit(xids[LATER]); + for (size_t k = 0; k < lengthof(cases); k++) + { + txn_visible_tests_written(io, state, cases[k].key, + txn_visible_tests_by(xids[cases[k].writer], 0, cases[k].command), k + 1); + if (cases[k].deleter >= 0) + txn_visible_tests_written(io, state, cases[k].key, + txn_visible_tests_by(xids[cases[k].deleter], 0, cases[k].deleted), 0); + } + taken = simc_historic(xids[LISTED], xids[LATER], &xids[LISTED], 1, &xids[OWN], 1, at_five); + lent = simc_Taken_lent(&taken); + for (size_t k = 0; k < lengthof(cases); k++) + { + uint64 tuple = k + 1; + + txn_visible_tests_named_sees_is(__LINE__, io, state, cases[k].key, &lent, &tuple, cases[k].seen ? 1 : 0); + } + earlier = simc_historic(xids[LISTED], xids[LATER], &xids[LISTED], 1, &xids[OWN], 1, at_three); + lent = simc_Taken_lent(&earlier); + txn_visible_tests_sees_is(__LINE__, io, state, "own deleted by own", &lent, fifteen, 1); + txn_visible_tests_sees_is(__LINE__, io, state, "own before", &lent, NULL, 0); + simc_Taken_drop(&earlier); + simc_Taken_drop(&taken); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* txn_visible_tests_conflicts returns the writers a serializable read of a key conflicts out with, newest first, each with its row's tuple id. */ +static StdVec +txn_visible_tests_conflicts(HostIoIo *io, HostSharedRecord state, const char *key, const HostSnapshot *snapshot, + const TxnVisibleSerial *serial) +{ + HostDeltaHold hold = txn_visible_tests_hold(io); + StdVec versions = txn_visible_tests_versioned(io, &hold, state, key); + TxnVisibleSeen seen = txn_visible_tests_seen_of(io, snapshot, &versions, serial); + StdVec conflicts; + + std_vec_new(&conflicts, sizeof(TxnVisibleTestsConflict), _Alignof(TxnVisibleTestsConflict)); + for (size_t k = 0; k < seen.conflicts.len; k++) + { + const txn_TupleRowXid_t *found = &STD_VEC_AT(&seen.conflicts, txn_TupleRowXid_t, k); + TxnVisibleTestsConflict conflict = {found->f0.f0, found->f1.f0}; + + std_vec_push_Checked(&conflicts, &conflict, sizeof(conflict), _Alignof(TxnVisibleTestsConflict)); + } + txn_visible_Seen_drop(&seen); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + return conflicts; +} + +/* txn_visible_tests_conflicts_is asserts at a line that a serializable read of a key conflicts out with exactly the writers given. */ +static void +txn_visible_tests_conflicts_is(uint32 line, HostIoIo *io, HostSharedRecord state, const char *key, + const SimcTaken *taken, const TxnVisibleSerial *serial, + const TxnVisibleTestsConflict *wanted, size_t wanted_len) +{ + HostSnapshot lent = simc_Taken_lent(taken); + StdVec got = txn_visible_tests_conflicts(io, state, key, &lent, serial); + StdVec want = {(void *) wanted, wanted_len, wanted_len}; + bool same = got.len == wanted_len; + + for (size_t k = 0; same && k < wanted_len; k++) + same = STD_VEC_AT(&got, TxnVisibleTestsConflict, k).tuple == wanted[k].tuple && + STD_VEC_AT(&got, TxnVisibleTestsConflict, k).xid == wanted[k].xid; + STD_ASSERT_EQ(line, 9, same, &got, txn_visible_tests_conflicts_Debug_fmt, &want, + txn_visible_tests_conflicts_Debug_fmt); + std_vec_free_Checked(&got, sizeof(TxnVisibleTestsConflict), _Alignof(TxnVisibleTestsConflict)); +} + +/* txn_visible_tests_base_conflict_is asserts at a line the writer a base row's serializable read conflicts out with, an xid of zero for none. */ +static void +txn_visible_tests_base_conflict_is(uint32 line, HostIoIo *io, const TxnVisibleSeen *seen, + const TxnVisibleSerial *serial, bool visible, HostXid wanted) +{ + bool has = false; + HostXid xid = {0}; + HostError error; + StdOptionRef got; + StdOptionRef want = {wanted.f0 != 0, STD_ITEM(&wanted, host_Xid_Debug_fmt)}; + HostRow nine = {9}; + + if (!txn_visible_Seen_base_conflict(seen, io, serial, nine, 2, visible, &has, &xid, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + got = (StdOptionRef) + { + has, STD_ITEM(&xid, host_Xid_Debug_fmt) + }; + STD_ASSERT_EQ(line, 9, has == (wanted.f0 != 0) && (!has || xid.f0 == wanted.f0), &got, std_option_Debug_fmt, + &want, std_option_Debug_fmt); +} + +/* + * test_txn_visible_tests_a_serializable_read_conflicts_out_with_the_writers_heaps_rule_names: + * HeapCheckForSerializableConflictOut's cases over the delta: a running + * writer's row the read doesn't see conflicts with its writer's top-level + * transaction; a committed row the read sees conflicts with nothing, and its + * running deleter's when the read sees the row it deletes; an aborted + * writer's row and the read's own conflict with nothing; a row its deleter + * committed below the read's xmin is dead and conflicts with nothing; and a + * base row a running deleter ends conflicts with that deleter while the read + * sees it. + */ +void +test_txn_visible_tests_a_serializable_read_conflicts_out_with_the_writers_heaps_rule_names(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9346); + HostSharedRecord state = host_shared_Held_record(&held); + const TxnVisibleSerial floor = {.xmin = {TXN_STATUS_FIRST_NORMAL}}; + TxnVisibleSerial own = floor; + TxnVisibleSerial past = {0}; + TxnVisibleSerial between = {0}; + HostXid top = simc_begin(); + HostXid sub = simc_begin_under(top); + HostXid writer; + HostXid deleter; + HostXid aborted; + HostXid ender; + HostXid early; + HostXid late; + SimcTaken during; + SimcTaken before; + SimcTaken after; + SimcTaken now; + HostSnapshot lent; + HostDeltaHold hold; + StdVec versions; + TxnVisibleSeen seen; + TxnVisibleTestsConflict running; + TxnVisibleTestsConflict deleting; + HostXid none = {0}; + + txn_visible_tests_written(io, state, "running", txn_visible_tests_by(sub, top.f0, 0), 1); + during = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + running = (TxnVisibleTestsConflict) + { + 1, top.f0 + }; + txn_visible_tests_conflicts_is(__LINE__, io, state, "running", &during, &floor, &running, 1); + own.has_own = true; + own.own = top; + txn_visible_tests_conflicts_is(__LINE__, io, state, "running", &during, &own, NULL, 0); + + writer = simc_begin(); + txn_visible_tests_written(io, state, "live", txn_visible_tests_by(writer, 0, 0), 2); + simc_commit(writer); + deleter = simc_begin(); + before = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + txn_visible_tests_conflicts_is(__LINE__, io, state, "live", &before, &floor, NULL, 0); + txn_visible_tests_written(io, state, "live", txn_visible_tests_by(deleter, 0, 0), 0); + deleting = (TxnVisibleTestsConflict) + { + 2, deleter.f0 + }; + txn_visible_tests_conflicts_is(__LINE__, io, state, "live", &before, &floor, &deleting, 1); + simc_commit(deleter); + past.xmin.f0 = deleter.f0 + 1; + txn_visible_tests_conflicts_is(__LINE__, io, state, "live", &before, &past, NULL, 0); + + aborted = simc_begin(); + txn_visible_tests_written(io, state, "aborted", txn_visible_tests_by(aborted, 0, 0), 3); + simc_abort(aborted); + after = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + txn_visible_tests_conflicts_is(__LINE__, io, state, "aborted", &after, &floor, NULL, 0); + + ender = simc_begin(); + txn_visible_tests_written(io, state, "based", txn_visible_tests_by(ender, 0, 0), 0); + hold = txn_visible_tests_hold(io); + versions = txn_visible_tests_versioned(io, &hold, state, "based"); + lent = simc_Taken_lent(&after); + seen = txn_visible_tests_seen_of(io, &lent, &versions, &floor); + txn_visible_tests_base_conflict_is(__LINE__, io, &seen, &floor, true, ender); + txn_visible_tests_base_conflict_is(__LINE__, io, &seen, &floor, false, none); + txn_visible_Seen_drop(&seen); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + + early = simc_begin(); + late = simc_begin(); + txn_visible_tests_written(io, state, "dead", txn_visible_tests_by(late, 0, 0), 4); + simc_commit(late); + txn_visible_tests_written(io, state, "dead", txn_visible_tests_by(early, 0, 0), 0); + simc_commit(early); + between.xmin.f0 = early.f0 + 1; + now = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + txn_visible_tests_conflicts_is(__LINE__, io, state, "dead", &now, &between, NULL, 0); + simc_Taken_drop(&now); + simc_Taken_drop(&after); + simc_Taken_drop(&before); + simc_Taken_drop(&during); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_visible_tests_a_transactions_own_row_is_seen_from_the_command_after_its_write: + * heap's MVCC rule over the caller's own write: a row the transaction wrote + * is seen by a snapshot whose command is past the write's, and not by one + * taken at the write's own command. + */ +void +test_txn_visible_tests_a_transactions_own_row_is_seen_from_the_command_after_its_write(void) +{ + static const uint64 one[] = {1}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9347); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid sub; + HostXid top; + HostCommand command; + HostError error; + + if (!host_current_writer(HOST_IO_IO_IO_DOORS(io), &sub, &top, &error) || + !host_current_command(HOST_IO_IO_IO_DOORS(io), true, &command, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_visible_tests_written(io, state, "own", txn_visible_tests_by(sub, top.f0, command.f0), 1); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "own", simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false), NULL, + 0); + simc_next_command(); + txn_visible_tests_taken_sees_is(__LINE__, io, state, "own", simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false), one, 1); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* TxnVisibleTestsXids is a slice of xids, as an answer's text shows them. */ +typedef struct TxnVisibleTestsXids +{ + const HostXid *ptr; + size_t len; +} TxnVisibleTestsXids; + +/* txn_visible_tests_xids_Debug_fmt is a slice of xids' Debug. */ +static bool +txn_visible_tests_xids_Debug_fmt(const void *value, StdFormatter *f) +{ + const TxnVisibleTestsXids *xids = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < xids->len; k++) + std_DebugList_entry(&list, &xids->ptr[k], host_Xid_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_visible_tests_format returns a String of a template's pieces and arguments, as format! makes it. */ +static StdString +txn_visible_tests_format(const StdFmtPiece *pieces, size_t pieces_len, const StdFmtArg *args) +{ + StdFmtArguments arguments = {pieces, pieces_len, args}; + StdString text; + + std_fmt_format_Checked(&text, &arguments); + return text; +} + +/* txn_visible_tests_answer_text returns an updatable answer's text as the case words it: its kind, the xids it names, and the one it is over. */ +static StdString +txn_visible_tests_answer_text(TxnVisibleUpdatableTag tag, const HostXid *by, size_t by_len, const HostXid *over) +{ + TxnVisibleTestsXids holders = {by, by_len}; + + switch (tag) + { + case TXN_VISIBLE_UPDATABLE_INVISIBLE: + { + StdFmtPiece pieces[] = {STD_FMT_LIT("invisible")}; + + return txn_visible_tests_format(pieces, lengthof(pieces), NULL); + } + case TXN_VISIBLE_UPDATABLE_SELF_MODIFIED: + { + StdFmtPiece pieces[] = {STD_FMT_LIT("self-modified by "), STD_FMT_ARG(0)}; + StdFmtArg args[] = {{by, host_Xid_Debug_fmt}}; + + return txn_visible_tests_format(pieces, lengthof(pieces), args); + } + case TXN_VISIBLE_UPDATABLE_BEING_MODIFIED: + { + StdFmtPiece pieces[] = {STD_FMT_LIT("being modified by "), STD_FMT_ARG(0), STD_FMT_LIT(" over "), + STD_FMT_ARG(1)}; + StdFmtArg args[] = {{&holders, txn_visible_tests_xids_Debug_fmt}, {over, host_Xid_Debug_fmt}}; + + return txn_visible_tests_format(pieces, over != NULL ? 4 : 2, args); + } + case TXN_VISIBLE_UPDATABLE_UPDATED: + { + StdFmtPiece pieces[] = {STD_FMT_LIT("updated by "), STD_FMT_ARG(0)}; + StdFmtArg args[] = {{by, host_Xid_Debug_fmt}}; + + return txn_visible_tests_format(pieces, lengthof(pieces), args); + } + case TXN_VISIBLE_UPDATABLE_DELETED: + { + StdFmtPiece pieces[] = {STD_FMT_LIT("deleted by "), STD_FMT_ARG(0)}; + StdFmtArg args[] = {{by, host_Xid_Debug_fmt}}; + + return txn_visible_tests_format(pieces, lengthof(pieces), args); + } + default: + { + StdFmtPiece pieces[] = {STD_FMT_LIT("ok")}; + + return txn_visible_tests_format(pieces, lengthof(pieces), NULL); + } + } +} + +/* txn_visible_tests_writer_xid returns the xid of a version's writer. */ +static HostXid +txn_visible_tests_writer_xid(const TxnMemtableVersionVersion *version) +{ + return txn_visible_tests_xid(0, version).changer; +} + +/* txn_visible_tests_row_of returns a version's row, panicking as Option::unwrap does when it writes none. */ +static TxnVisibleRowVersion +txn_visible_tests_row_of(TxnMemtableVersionVersion *version, uint32 line) +{ + TxnVisibleRowVersion row; + + if (!txn_visible_RowVersion_of(version, &row)) + std_panic(RUST_FILE, line, 0, "called `Option::unwrap()` on a `None` value", + sizeof("called `Option::unwrap()` on a `None` value") - 1); + return row; +} + +/* txn_visible_tests_updatable returns what updatable answers of a change of a row over versions, at command 1. */ +static TxnVisibleUpdatable +txn_visible_tests_updatable(HostIoIo *io, StdVec *versions, TxnVisibleChanged changed) +{ + TxnVisibleUpdatable answer; + HostError error; + + if (!txn_visible_updatable(io, versions->ptr, versions->len, changed, 1, &answer, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return answer; +} + +/* + * txn_visible_tests_asked returns what updatable answers of a key's row as + * the case words it: its oldest row version's unless based, the newest + * version's when based with tuple id zero, and the base's row of the tuple + * id when based with any other. + */ +static StdString +txn_visible_tests_asked(HostIoIo *io, HostSharedRecord state, const char *key, bool based, uint64 tuple) +{ + HostDeltaHold hold = txn_visible_tests_hold(io); + StdVec versions = txn_visible_tests_versioned(io, &hold, state, key); + TxnMemtableVersionVersion *all = versions.ptr; + TxnVisibleChanged changed = {.tag = TXN_VISIBLE_CHANGED_DELTA}; + TxnVisibleUpdatable answer; + StdVec by; + HostXid xid; + HostXid over; + bool has_over = false; + StdString text; + + if (based && tuple == 0) + changed.u.delta.row = txn_visible_tests_row_of(&all[0], __LINE__); + else if (based) + { + changed.tag = TXN_VISIBLE_CHANGED_BASE; + changed.u.base.f0 = tuple; + } + else + { + size_t at = versions.len; + bool found = false; + + while (!found && at-- > 0) + found = txn_visible_RowVersion_of(&all[at], &changed.u.delta.row); + if (!found) + std_panic(RUST_FILE, __LINE__, 0, "called `Option::unwrap()` on a `None` value", + sizeof("called `Option::unwrap()` on a `None` value") - 1); + changed.u.delta.at = at; + } + answer = txn_visible_tests_updatable(io, &versions, changed); + std_vec_new(&by, sizeof(HostXid), _Alignof(HostXid)); + switch (answer.tag) + { + case TXN_VISIBLE_UPDATABLE_SELF_MODIFIED: + xid = txn_visible_tests_writer_xid(answer.u.self_modified); + std_vec_push_Checked(&by, &xid, sizeof(xid), _Alignof(HostXid)); + break; + case TXN_VISIBLE_UPDATABLE_BEING_MODIFIED: + for (size_t k = 0; k < answer.u.being_modified.holders.len; k++) + { + xid = txn_visible_tests_writer_xid(STD_VEC_AT(&answer.u.being_modified.holders, + TxnMemtableVersionVersion *, k)); + std_vec_push_Checked(&by, &xid, sizeof(xid), _Alignof(HostXid)); + } + has_over = answer.u.being_modified.ended != NULL; + if (has_over) + over = txn_visible_tests_writer_xid(answer.u.being_modified.ended); + break; + case TXN_VISIBLE_UPDATABLE_UPDATED: + xid = txn_visible_tests_writer_xid(answer.u.updated); + std_vec_push_Checked(&by, &xid, sizeof(xid), _Alignof(HostXid)); + break; + case TXN_VISIBLE_UPDATABLE_DELETED: + xid = txn_visible_tests_writer_xid(answer.u.deleted); + std_vec_push_Checked(&by, &xid, sizeof(xid), _Alignof(HostXid)); + break; + default: + break; + } + text = txn_visible_tests_answer_text(answer.tag, by.ptr, by.len, has_over ? &over : NULL); + + std_vec_free_Checked(&by, sizeof(HostXid), _Alignof(HostXid)); + + txn_visible_Updatable_drop(&answer); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + return text; +} + +/* txn_visible_tests_asked_is asserts at a line that an answer's text is the one wanted, naming the key, freeing both. */ +static void +txn_visible_tests_asked_is(uint32 line, StdString got, StdString wanted, const char *key) +{ + if (got.len != wanted.len || (got.len != 0 && memcmp(got.ptr, wanted.ptr, got.len) != 0)) + { + StdStr name = {key, key != NULL ? strlen(key) : 0}; + StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&name, std_str_Display_fmt}}; + StdFmtArguments arguments = {pieces, 1, args}; + + std_assert_failed(RUST_FILE, line, 13, "==", &got, std_String_Debug_fmt, &wanted, std_String_Debug_fmt, + key != NULL ? &arguments : NULL); + } + std_String_drop(&got); + std_String_drop(&wanted); +} + +/* The writers of the updatable case, by their places among its xids. */ +enum +{ + TXN_VISIBLE_TESTS_COMMITTED, + TXN_VISIBLE_TESTS_ABORTED, + TXN_VISIBLE_TESTS_RUNNING, + TXN_VISIBLE_TESTS_LOCKER, + TXN_VISIBLE_TESTS_OWN, +}; + +/* TxnVisibleTestsWrite is a version the updatable case pushes: its writer's place, its command, and its op, of a tuple id or a lock's mode. */ +typedef struct TxnVisibleTestsWrite +{ + int who; + uint32 command; + TxnMemtableOpTag op; + uint64 tuple; + TxnMemtableLockMode mode; +} TxnVisibleTestsWrite; + +/* TxnVisibleTestsKeyed is a key of the updatable case and its versions, oldest first. */ +typedef struct TxnVisibleTestsKeyed +{ + const char *key; + TxnVisibleTestsWrite writes[3]; + size_t len; +} TxnVisibleTestsKeyed; + +/* TxnVisibleTestsAnswer is an answer the updatable case wants of a key: its kind, the writer it names or -1, and the one it is over or -1. */ +typedef struct TxnVisibleTestsAnswer +{ + const char *key; + TxnVisibleUpdatableTag tag; + int by; + int over; +} TxnVisibleTestsAnswer; + +/* txn_visible_tests_op returns a case's version's op. */ +static TxnMemtableOp +txn_visible_tests_op(const TxnVisibleTestsWrite *write) +{ + TxnMemtableOp op = {.tag = write->op}; + + switch (write->op) + { + case TXN_MEMTABLE_OP_PUT: + op.u.put = txn_visible_tests_put(write->tuple); + break; + case TXN_MEMTABLE_OP_LOCK: + op.u.lock = write->mode; + break; + case TXN_MEMTABLE_OP_MOVE: + op.u.move.tag = TXN_MEMTABLE_SUCCESSOR_OTHER_TABLE; + break; + case TXN_MEMTABLE_OP_ADD_ROW: + op.u.add_row = txn_visible_tests_put(write->tuple); + break; + case TXN_MEMTABLE_OP_DELETE_ROW: + op.u.delete_row.f0.f0 = write->tuple; + break; + case TXN_MEMTABLE_OP_LOCK_ROW: + op.u.lock_row.f0.f0 = write->tuple; + op.u.lock_row.f1 = write->mode; + break; + default: + break; + } + return op; +} + +/* txn_visible_tests_wanted returns an answer's wanted text, by the case's xids. */ +static StdString +txn_visible_tests_wanted(TxnVisibleUpdatableTag tag, const HostXid *xids, int by, int over) +{ + return txn_visible_tests_answer_text(tag, by >= 0 ? &xids[by] : NULL, by >= 0 ? 1 : 0, + over >= 0 ? &xids[over] : NULL); +} + +/* + * test_txn_visible_tests_a_rows_change_is_answered_as_heaps_satisfies_update_answers: + * (4b)'s updatable (heap's HeapTupleSatisfiesUpdate): a row whose writer + * aborted or another transaction runs is invisible, and the caller's own is + * invisible at or past its command and changeable before it; a committed row + * with nothing above it, or only an aborted change or a committed lock, is + * changeable; another transaction's running change or lock is changing it, + * and so is the caller's own lock, each by its version; a committed update or + * move updated it and a committed delete deleted it, each by its version, + * unless a locker below it still runs, which is changing it over that + * version; the caller's own change at or past its command self-modified it + * and before it hid it; a version naming another row of the key changes + * nothing; a base row is asked the same way; and a row a no-key update wrote + * over a running key-share locker's row is held by the lock it carried. + */ +void +test_txn_visible_tests_a_rows_change_is_answered_as_heaps_satisfies_update_answers(void) +{ + static const TxnVisibleTestsKeyed rows[] = { + {"aborted", {{TXN_VISIBLE_TESTS_ABORTED, 0, TXN_MEMTABLE_OP_PUT, 1}}, 1}, + {"running", {{TXN_VISIBLE_TESTS_RUNNING, 0, TXN_MEMTABLE_OP_PUT, 2}}, 1}, + {"own late", {{TXN_VISIBLE_TESTS_OWN, 1, TXN_MEMTABLE_OP_PUT, 3}}, 1}, + {"own early", {{TXN_VISIBLE_TESTS_OWN, 0, TXN_MEMTABLE_OP_PUT, 4}}, 1}, + {"alone", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 5}}, 1}, + {"undone", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 6}, + {TXN_VISIBLE_TESTS_ABORTED, 0, TXN_MEMTABLE_OP_DELETE}}, 2}, + {"unlocked", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 7}, + {TXN_VISIBLE_TESTS_COMMITTED, 1, TXN_MEMTABLE_OP_LOCK, 0, TXN_MEMTABLE_LOCK_MODE_UPDATE}}, 2}, + {"changing", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 8}, + {TXN_VISIBLE_TESTS_RUNNING, 0, TXN_MEMTABLE_OP_PUT, 9}}, 2}, + {"locking", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 10}, + {TXN_VISIBLE_TESTS_LOCKER, 0, TXN_MEMTABLE_OP_LOCK, 0, TXN_MEMTABLE_LOCK_MODE_SHARE}}, 2}, + {"own lock", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 11}, + {TXN_VISIBLE_TESTS_OWN, 0, TXN_MEMTABLE_OP_LOCK, 0, TXN_MEMTABLE_LOCK_MODE_SHARE}}, 2}, + {"updated", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 12}, + {TXN_VISIBLE_TESTS_COMMITTED, 1, TXN_MEMTABLE_OP_PUT, 13}}, 2}, + {"deleted", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 14}, + {TXN_VISIBLE_TESTS_COMMITTED, 1, TXN_MEMTABLE_OP_DELETE}}, 2}, + {"moved", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 15}, + {TXN_VISIBLE_TESTS_COMMITTED, 1, TXN_MEMTABLE_OP_MOVE}}, 2}, + {"shared", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 16}, + {TXN_VISIBLE_TESTS_LOCKER, 0, TXN_MEMTABLE_OP_LOCK, 0, TXN_MEMTABLE_LOCK_MODE_KEY_SHARE}, + {TXN_VISIBLE_TESTS_COMMITTED, 1, TXN_MEMTABLE_OP_PUT, 17}}, 3}, + {"own late change", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 18}, + {TXN_VISIBLE_TESTS_OWN, 1, TXN_MEMTABLE_OP_DELETE}}, 2}, + {"own early change", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 19}, + {TXN_VISIBLE_TESTS_OWN, 0, TXN_MEMTABLE_OP_DELETE}}, 2}, + {"other row", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_ADD_ROW, 20}, + {TXN_VISIBLE_TESTS_RUNNING, 0, TXN_MEMTABLE_OP_DELETE_ROW, 21}}, 2}, + {"base", {{TXN_VISIBLE_TESTS_RUNNING, 0, TXN_MEMTABLE_OP_LOCK_ROW, 22, TXN_MEMTABLE_LOCK_MODE_UPDATE}}, 1}, + {"carried", {{TXN_VISIBLE_TESTS_COMMITTED, 0, TXN_MEMTABLE_OP_PUT, 24}, + {TXN_VISIBLE_TESTS_LOCKER, 0, TXN_MEMTABLE_OP_LOCK, 0, TXN_MEMTABLE_LOCK_MODE_KEY_SHARE}, + {TXN_VISIBLE_TESTS_COMMITTED, 1, TXN_MEMTABLE_OP_PUT, 25}}, 3}, + }; + static const TxnVisibleTestsAnswer answers[] = { + {"aborted", TXN_VISIBLE_UPDATABLE_INVISIBLE, -1, -1}, + {"running", TXN_VISIBLE_UPDATABLE_INVISIBLE, -1, -1}, + {"own late", TXN_VISIBLE_UPDATABLE_INVISIBLE, -1, -1}, + {"own early", TXN_VISIBLE_UPDATABLE_OK, -1, -1}, + {"alone", TXN_VISIBLE_UPDATABLE_OK, -1, -1}, + {"undone", TXN_VISIBLE_UPDATABLE_OK, -1, -1}, + {"unlocked", TXN_VISIBLE_UPDATABLE_OK, -1, -1}, + {"changing", TXN_VISIBLE_UPDATABLE_BEING_MODIFIED, TXN_VISIBLE_TESTS_RUNNING, -1}, + {"locking", TXN_VISIBLE_UPDATABLE_BEING_MODIFIED, TXN_VISIBLE_TESTS_LOCKER, -1}, + {"own lock", TXN_VISIBLE_UPDATABLE_BEING_MODIFIED, TXN_VISIBLE_TESTS_OWN, -1}, + {"updated", TXN_VISIBLE_UPDATABLE_UPDATED, TXN_VISIBLE_TESTS_COMMITTED, -1}, + {"deleted", TXN_VISIBLE_UPDATABLE_DELETED, TXN_VISIBLE_TESTS_COMMITTED, -1}, + {"moved", TXN_VISIBLE_UPDATABLE_UPDATED, TXN_VISIBLE_TESTS_COMMITTED, -1}, + {"shared", TXN_VISIBLE_UPDATABLE_BEING_MODIFIED, TXN_VISIBLE_TESTS_LOCKER, TXN_VISIBLE_TESTS_COMMITTED}, + {"own late change", TXN_VISIBLE_UPDATABLE_SELF_MODIFIED, TXN_VISIBLE_TESTS_OWN, -1}, + {"own early change", TXN_VISIBLE_UPDATABLE_INVISIBLE, -1, -1}, + {"other row", TXN_VISIBLE_UPDATABLE_OK, -1, -1}, + }; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9349); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid xids[5]; + HostXid top; + HostError error; + + xids[TXN_VISIBLE_TESTS_COMMITTED] = simc_begin(); + xids[TXN_VISIBLE_TESTS_ABORTED] = simc_begin(); + xids[TXN_VISIBLE_TESTS_RUNNING] = simc_begin(); + xids[TXN_VISIBLE_TESTS_LOCKER] = simc_begin(); + if (!host_current_writer(HOST_IO_IO_IO_DOORS(io), &xids[TXN_VISIBLE_TESTS_OWN], &top, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + for (size_t k = 0; k < lengthof(rows); k++) + for (size_t j = 0; j < rows[k].len; j++) + txn_visible_tests_pushing(io, state, rows[k].key, + txn_visible_tests_by(xids[rows[k].writes[j].who], 0, rows[k].writes[j].command), + txn_visible_tests_op(&rows[k].writes[j])); + simc_abort(xids[TXN_VISIBLE_TESTS_ABORTED]); + simc_commit(xids[TXN_VISIBLE_TESTS_COMMITTED]); + for (size_t k = 0; k < lengthof(answers); k++) + txn_visible_tests_asked_is(__LINE__, txn_visible_tests_asked(io, state, answers[k].key, false, 0), + txn_visible_tests_wanted(answers[k].tag, xids, answers[k].by, answers[k].over), + answers[k].key); + txn_visible_tests_asked_is(__LINE__, txn_visible_tests_asked(io, state, "base", true, 22), + txn_visible_tests_wanted(TXN_VISIBLE_UPDATABLE_BEING_MODIFIED, xids, + TXN_VISIBLE_TESTS_RUNNING, -1), NULL); + txn_visible_tests_asked_is(__LINE__, txn_visible_tests_asked(io, state, "base", true, 23), + txn_visible_tests_wanted(TXN_VISIBLE_UPDATABLE_OK, xids, -1, -1), NULL); + txn_visible_tests_asked_is(__LINE__, txn_visible_tests_asked(io, state, "carried", true, 0), + txn_visible_tests_wanted(TXN_VISIBLE_UPDATABLE_BEING_MODIFIED, xids, + TXN_VISIBLE_TESTS_LOCKER, -1), NULL); + simc_commit(xids[TXN_VISIBLE_TESTS_LOCKER]); + txn_visible_tests_asked_is(__LINE__, txn_visible_tests_asked(io, state, "shared", false, 0), + txn_visible_tests_wanted(TXN_VISIBLE_UPDATABLE_UPDATED, xids, + TXN_VISIBLE_TESTS_COMMITTED, -1), NULL); + txn_visible_tests_asked_is(__LINE__, txn_visible_tests_asked(io, state, "locking", false, 0), + txn_visible_tests_wanted(TXN_VISIBLE_UPDATABLE_OK, xids, -1, -1), NULL); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* txn_visible_tests_resolved_is asserts at a line what an immediate key's resolve finds over versions, freeing the wanted answer. */ +static void +txn_visible_tests_resolved_is(uint32 line, HostIoIo *io, StdVec *versions, TxnVisibleResolved *wanted) +{ + StdBytes none = {NULL, 0}; + TxnVisibleResolved got; + HostError error; + + if (!txn_visible_resolved(io, versions->ptr, versions->len, none, &got, &error)) + STD_RESULT_EXPECT_FAILED(line, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT_EQ(line, 9, txn_visible_Resolved_PartialEq_eq(&got, wanted), &got, txn_visible_Resolved_Debug_fmt, + wanted, txn_visible_Resolved_Debug_fmt); + txn_visible_Resolved_drop(&got); + txn_visible_Resolved_drop(wanted); +} + +/* + * test_txn_visible_tests_a_killed_speculative_insert_is_aborted_for_every_reader_at_once: + * (4b)'s speculative insert, CommPF 7's (b): a running transaction's insert + * of a key, killed by its writer, is aborted for every reader at once, its + * transaction still running: the resolve passes over it, finding the key free + * with no wait, a Dirty read sees no row and reports nothing running, and its + * outcome is aborted. + */ +void +test_txn_visible_tests_a_killed_speculative_insert_is_aborted_for_every_reader_at_once(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9350); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid running = simc_begin(); + HostDeltaHold hold; + StdVec versions; + TxnVisibleResolved wait = {.tag = TXN_VISIBLE_RESOLVED_WAIT}; + TxnVisibleResolved free_key = {.tag = TXN_VISIBLE_RESOLVED_FREE}; + HostStatus status; + SimcTaken dirty; + HostSnapshot lent; + TxnVisibleSeen seen; + HostError error; + + txn_visible_tests_written(io, state, "speculative", txn_visible_tests_by(running, 0, 0), 1); + hold = txn_visible_tests_hold(io); + versions = txn_visible_tests_versioned(io, &hold, state, "speculative"); + wait.u.wait.f0 = running; + wait.u.wait.has_f1 = true; + wait.u.wait.f1.f0 = 1; + txn_visible_tests_resolved_is(__LINE__, io, &versions, &wait); + txn_memtable_version_Version_kill(&STD_VEC_AT(&versions, TxnMemtableVersionVersion, 0)); + txn_visible_tests_resolved_is(__LINE__, io, &versions, &free_key); + if (!txn_outcome_outcome(io, &STD_VEC_AT(&versions, TxnMemtableVersionVersion, 0), &status, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, status == HOST_STATUS_ABORTED, "outcome(io, &versions[0]).unwrap() == Status::Aborted"); + dirty = simc_snapshot(HOST_SNAPSHOT_KIND_DIRTY, false); + lent = simc_Taken_lent(&dirty); + seen = txn_visible_tests_seen_of(io, &lent, &versions, NULL); + STD_ASSERT(__LINE__, 9, seen.rows.len == 0, "seen.rows.is_empty()"); + txn_visible_Seen_drop(&seen); + simc_Taken_drop(&dirty); + txn_memtable_version_versions_drop(&versions, 0); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* txn_visible_tests_handles_Debug_fmt is a Vec of handles' Debug. */ +static bool +txn_visible_tests_handles_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *handles = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < handles->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(handles, TxnMemtableVersionHandle, k), + txn_memtable_version_Handle_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_visible_tests_strings_Debug_fmt is a Vec's Debug. */ +static bool +txn_visible_tests_strings_Debug_fmt(const void *value, StdFormatter *f) +{ + const StdVec *strings = value; + StdDebugList list = std_fmt_debug_list(f); + + for (size_t k = 0; k < strings->len; k++) + std_DebugList_entry(&list, &STD_VEC_AT(strings, StdString, k), std_String_Debug_fmt); + return std_DebugList_finish(&list); +} + +/* txn_visible_tests_strings_drop frees a Vec. */ +static void +txn_visible_tests_strings_drop(StdVec *strings) +{ + for (size_t k = 0; k < strings->len; k++) + std_String_drop(&STD_VEC_AT(strings, StdString, k)); + std_vec_free_Checked(strings, sizeof(StdString), _Alignof(StdString)); +} + +/* txn_visible_tests_handle_push appends a version's handle, if there is a version. */ +static void +txn_visible_tests_handle_push(StdVec *handles, const TxnMemtableVersionVersion *version) +{ + if (version != NULL) + std_vec_push_Checked(handles, &version->handle, sizeof(version->handle), _Alignof(TxnMemtableVersionHandle)); +} + +/* txn_visible_tests_sighted returns a row's sight as answers words it: its dirty report, its changer's handle, and its commander's. */ +static StdString +txn_visible_tests_sighted(bool base, bool kept, const TxnVisibleDirty *dirty, const TxnMemtableVersionVersion *changer, + const TxnMemtableVersionVersion *commander) +{ + TxnMemtableVersionHandle changed = {changer != NULL ? changer->handle.f0 : 0}; + TxnMemtableVersionHandle commanded = {commander != NULL ? commander->handle.f0 : 0}; + StdOptionRef changer_shown = {changer != NULL, STD_ITEM(&changed, txn_memtable_version_Handle_Debug_fmt)}; + StdOptionRef commander_shown = {commander != NULL, STD_ITEM(&commanded, txn_memtable_version_Handle_Debug_fmt)}; + StdFmtPiece row_pieces[] = {STD_FMT_ARG(0), STD_FMT_LIT(" "), STD_FMT_ARG(1), STD_FMT_LIT(" "), STD_FMT_ARG(2)}; + StdFmtPiece base_pieces[] = {STD_FMT_LIT("base "), STD_FMT_ARG(3), STD_FMT_LIT(" "), STD_FMT_ARG(0), + STD_FMT_LIT(" "), STD_FMT_ARG(1), STD_FMT_LIT(" "), STD_FMT_ARG(2)}; + StdFmtArg args[] = {{dirty, txn_visible_Dirty_Debug_fmt}, {&changer_shown, std_option_Debug_fmt}, + {&commander_shown, std_option_Debug_fmt}, {&kept, std_bool_Display_fmt}}; + + return base ? txn_visible_tests_format(base_pieces, lengthof(base_pieces), args) : + txn_visible_tests_format(row_pieces, lengthof(row_pieces), args); +} + +/* + * txn_visible_tests_answers returns what seen under each of a case's + * snapshots and updatable answer a tuple id over versions newest first, with + * the row's place among them when they hold it: the row's sight, or the base + * row's when they don't hold it, and updatable's shape, in a Vec. + */ +static StdVec +txn_visible_tests_answers(HostIoIo *io, const HostSnapshot *snapshots, size_t snapshots_len, StdVec *versions, + bool has_row, size_t row, uint64 tuple) +{ + TxnVisibleChanged changed = {.tag = TXN_VISIBLE_CHANGED_BASE, .u.base = {tuple}}; + TxnVisibleUpdatable answer; + StdVec got; + StdVec handles; + uint8 kind; + StdString text; + + std_vec_new(&got, sizeof(StdString), _Alignof(StdString)); + for (size_t s = 0; s < snapshots_len; s++) + { + TxnVisibleRule rule = {&TXN_VISIBLE_SNAPSHOT_RULE_OPS, &snapshots[s]}; + TxnVisibleSeen seen = txn_visible_tests_seen_of(io, &snapshots[s], versions, NULL); + + for (size_t k = 0; k < seen.rows.len; k++) + { + const TxnVisibleSeenRow *one = &STD_VEC_AT(&seen.rows, TxnVisibleSeenRow, k); + + if (one->row.put.tuple.f0 != tuple) + continue; + text = txn_visible_tests_sighted(false, false, &one->dirty, one->changer, one->commander); + + std_vec_push_Checked(&got, &text, sizeof(text), _Alignof(StdString)); + } + if (!has_row) + { + bool kept; + TxnVisibleDirty dirty; + TxnMemtableVersionVersion *changer; + TxnMemtableVersionVersion *commander; + HostRow based = {tuple}; + HostError error; + + if (!txn_visible_Seen_base(&seen, io, rule, based, &kept, &dirty, &changer, &commander, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + text = txn_visible_tests_sighted(true, kept, &dirty, changer, commander); + + std_vec_push_Checked(&got, &text, sizeof(text), _Alignof(StdString)); + } + txn_visible_Seen_drop(&seen); + } + if (has_row) + { + changed.tag = TXN_VISIBLE_CHANGED_DELTA; + changed.u.delta.at = row; + changed.u.delta.row = txn_visible_tests_row_of(&STD_VEC_AT(versions, TxnMemtableVersionVersion, row), + __LINE__); + } + answer = txn_visible_tests_updatable(io, versions, changed); + std_vec_new(&handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + kind = (uint8) answer.tag; + if (answer.tag == TXN_VISIBLE_UPDATABLE_SELF_MODIFIED) + txn_visible_tests_handle_push(&handles, answer.u.self_modified); + else if (answer.tag == TXN_VISIBLE_UPDATABLE_BEING_MODIFIED) + { + for (size_t k = 0; k < answer.u.being_modified.holders.len; k++) + txn_visible_tests_handle_push(&handles, STD_VEC_AT(&answer.u.being_modified.holders, + TxnMemtableVersionVersion *, k)); + txn_visible_tests_handle_push(&handles, answer.u.being_modified.ended); + } + else if (answer.tag == TXN_VISIBLE_UPDATABLE_UPDATED) + txn_visible_tests_handle_push(&handles, answer.u.updated); + else if (answer.tag == TXN_VISIBLE_UPDATABLE_DELETED) + txn_visible_tests_handle_push(&handles, answer.u.deleted); + { + StdTupleRef shape = STD_TUPLE(2, STD_ITEM(&kind, std_u8_Debug_fmt), + STD_ITEM(&handles, txn_visible_tests_handles_Debug_fmt)); + StdFmtPiece pieces[] = {STD_FMT_ARG(0)}; + StdFmtArg args[] = {{&shape, std_tuple_Debug_fmt}}; + + text = txn_visible_tests_format(pieces, lengthof(pieces), args); + } + std_vec_push_Checked(&got, &text, sizeof(text), _Alignof(StdString)); + std_vec_free_Checked(&handles, sizeof(TxnMemtableVersionHandle), _Alignof(TxnMemtableVersionHandle)); + txn_visible_Updatable_drop(&answer); + return got; +} + +/* txn_visible_tests_strings_eq returns whether two Vecs hold the same texts. */ +static bool +txn_visible_tests_strings_eq(const StdVec *a, const StdVec *b) +{ + bool same = a->len == b->len; + + for (size_t k = 0; same && k < a->len; k++) + { + const StdString *x = &STD_VEC_AT(a, StdString, k); + const StdString *y = &STD_VEC_AT(b, StdString, k); + + same = x->len == y->len && (x->len == 0 || memcmp(x->ptr, y->ptr, x->len) == 0); + } + return same; +} + +/* txn_visible_tests_whole_at returns the place of a tuple id's row among a key's whole chain, if it holds it. */ +static bool +txn_visible_tests_whole_at(StdVec *whole, uint64 tuple, size_t *at) +{ + for (size_t k = 0; k < whole->len; k++) + { + TxnVisibleRowVersion row; + + if (txn_visible_RowVersion_of(&STD_VEC_AT(whole, TxnMemtableVersionVersion, k), &row) && + row.put.tuple.f0 == tuple) + { + *at = k; + return true; + } + } + return false; +} + +/* txn_visible_tests_slice returns a row's slice of a key's versions, as row_versions gives it. */ +static TxnMemtableSliceSlice +txn_visible_tests_slice(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, uint64 tuple) +{ + TxnMemtableSliceSlice slice; + HostRow tid = {tuple}; + HostError error; + + if (!txn_memtable_slice_row_versions(io, hold, state, (const uint8 *) "k", 1, tid, 0, &slice, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + return slice; +} + +/* txn_visible_tests_next returns the next of a xorshift generator's numbers below n. */ +static uint64 +txn_visible_tests_next(uint64 *seed, uint64 n) +{ + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + return *seed % n; +} + +/* txn_visible_tests_successor returns a delete's successor, the key "k" with a tuple id's eight big-endian bytes. */ +static TxnMemtableSuccessor +txn_visible_tests_successor(uint64 tid) +{ + TxnMemtableSuccessor successor = {.tag = TXN_MEMTABLE_SUCCESSOR_KEY}; + uint8 key[9] = {'k'}; + + for (int k = 0; k < 8; k++) + key[1 + k] = (uint8) (tid >> (8 * (7 - k))); + std_slice_to_vec_Checked(&successor.u.key, key, sizeof(key), 1, 1); + return successor; +} + +/* txn_visible_tests_random_push pushes one random version of the slice case's key, as its match chooses, by a writer. */ +static void +txn_visible_tests_random_push(HostIoIo *io, HostSharedRecord state, TxnMemtableVersionWriter writer, uint64 *seed, + uint64 *tid) +{ + bool named = *tid > 0; + uint64 row = named ? 1 + txn_visible_tests_next(seed, *tid) : 0; + uint64 choice = txn_visible_tests_next(seed, 8); + TxnMemtableOp op = {0}; + + if (choice == 0) + { + op.tag = TXN_MEMTABLE_OP_PUT; + op.u.put = txn_visible_tests_put(++*tid); + } + else if (choice == 1) + op.tag = TXN_MEMTABLE_OP_DELETE; + else if (choice == 2) + { + op.tag = TXN_MEMTABLE_OP_LOCK; + op.u.lock = TXN_MEMTABLE_LOCK_MODE_SHARE; + } + else if (choice == 3 && named) + { + op.tag = TXN_MEMTABLE_OP_DELETE_ROW; + op.u.delete_row.f0.f0 = row; + } + else if (choice == 4 && named) + { + op.tag = TXN_MEMTABLE_OP_DELETE_ROW; + op.u.delete_row.f0.f0 = row; + op.u.delete_row.has_f1 = true; + op.u.delete_row.f1 = txn_visible_tests_successor(++*tid); + txn_visible_tests_pushing(io, state, "k", writer, op); + op = (TxnMemtableOp) + { + .tag = TXN_MEMTABLE_OP_ADD_ROW, .u.add_row = txn_visible_tests_put(*tid) + }; + } + else if (choice == 5 && named) + { + op.tag = TXN_MEMTABLE_OP_UPDATE_ROW; + op.u.update_row.f0.f0 = row; + op.u.update_row.f1 = txn_visible_tests_put(++*tid); + } + else if (choice == 6 && named) + { + op.tag = TXN_MEMTABLE_OP_LOCK_ROW; + op.u.lock_row.f0.f0 = row; + op.u.lock_row.f1 = TXN_MEMTABLE_LOCK_MODE_KEY_SHARE; + } + else + { + op.tag = TXN_MEMTABLE_OP_ADD_ROW; + op.u.add_row = txn_visible_tests_put(++*tid); + } + txn_visible_tests_pushing(io, state, "k", writer, op); +} + +/* txn_visible_tests_walked_failed panics as the slice case's assert_eq! does, naming its case, tuple id, and walk. */ +static void +txn_visible_tests_walked_failed(const StdVec *got, const StdVec *expected, uint32 case_number, uint64 tuple, + uint32 walk, uint32 line) +{ + StdFmtPiece pieces[] = {STD_FMT_LIT("case "), STD_FMT_ARG(0), STD_FMT_LIT(", tuple "), STD_FMT_ARG(1), + STD_FMT_LIT(", walk "), STD_FMT_ARG(2)}; + StdFmtArg args[] = {{&case_number, std_u32_Display_fmt}, {&tuple, std_u64_Display_fmt}, + {&walk, std_u32_Display_fmt}}; + StdFmtArguments arguments = {pieces, lengthof(pieces), args}; + + std_assert_failed(RUST_FILE, line, 25, "==", got, txn_visible_tests_strings_Debug_fmt, expected, + txn_visible_tests_strings_Debug_fmt, &arguments); +} + +/* + * test_txn_visible_tests_a_rows_slice_answers_as_its_whole_chain: bugs row + * 134: over random chains, seen and updatable answer every tuple id the same + * over the slice row_versions gives as over the key's whole chain, at the + * key's first walk, its second, and after more versions are pushed, with + * writers committed, aborted, running, and the actor's own, its + * subtransactions' among them, one rolled back between two walks. + */ +void +test_txn_visible_tests_a_rows_slice_answers_as_its_whole_chain(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + uint64 seed = UINT64CONST(0x2545f4914f6cdd1d); + + simc_end(false); + for (uint32 case_number = 0; case_number < 60; case_number++) + { + HostSharedHeld held = txn_visible_tests_made(io, 9500 + case_number); + HostSharedRecord state = host_shared_Held_record(&held); + uint64 tid = 0; + + for (int round = 0; round < 3; round++) + { + uint64 pushes = 1 + txn_visible_tests_next(&seed, 12); + SimcTaken mvcc; + SimcTaken dirty; + HostSnapshot snapshots[2]; + HostDeltaHold hold; + StdVec whole; + + for (uint64 push = 0; push < pushes; push++) + { + bool own = txn_visible_tests_next(&seed, 5) == 0; + bool sub = txn_visible_tests_next(&seed, 7) == 0; + TxnMemtableVersionWriter writer; + uint64 ending; + HostXid begun; + + if (sub) + simc_savepoint(); + if (own || sub) + { + HostXid xid; + HostXid top; + HostError error; + + if (!host_current_writer(HOST_IO_IO_IO_DOORS(io), &xid, &top, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", + &error, host_Error_Debug_fmt); + writer = txn_visible_tests_by(xid, xid.f0 != top.f0 ? top.f0 : 0, 0); + } + else + writer = txn_visible_tests_by(simc_begin(), 0, 0); + txn_visible_tests_random_push(io, state, writer, &seed, &tid); + ending = txn_visible_tests_next(&seed, 3); + begun.f0 = writer.xid; + if (sub && ending == 0) + simc_rollback_to(); + else if (sub) + simc_release(); + else if (!own && ending == 0) + simc_commit(begun); + else if (!own && ending == 1) + simc_abort(begun); + } + mvcc = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + dirty = simc_snapshot(HOST_SNAPSHOT_KIND_DIRTY, false); + snapshots[0] = simc_Taken_lent(&mvcc); + snapshots[1] = simc_Taken_lent(&dirty); + hold = txn_visible_tests_hold(io); + whole = txn_visible_tests_versioned(io, &hold, state, "k"); + for (uint64 tuple = 1; tuple <= tid + 1; tuple++) + { + size_t at = 0; + bool has_at = txn_visible_tests_whole_at(&whole, tuple, &at); + StdVec expected = txn_visible_tests_answers(io, snapshots, lengthof(snapshots), &whole, has_at, + at, tuple); + + for (uint32 walk = 0; walk < 2; walk++) + { + TxnMemtableSliceSlice slice = txn_visible_tests_slice(io, &hold, state, tuple); + StdVec got = txn_visible_tests_answers(io, snapshots, lengthof(snapshots), &slice.versions, + slice.has_row, slice.row, tuple); + + if (!txn_visible_tests_strings_eq(&got, &expected)) + txn_visible_tests_walked_failed(&got, &expected, case_number, tuple, walk, __LINE__); + txn_visible_tests_strings_drop(&got); + txn_memtable_slice_Slice_drop(&slice); + } + txn_visible_tests_strings_drop(&expected); + } + txn_memtable_version_versions_drop(&whole, 0); + host_delta_Hold_drop(&hold); + simc_Taken_drop(&dirty); + simc_Taken_drop(&mvcc); + } + host_shared_Held_drop(&held); + simc_end(false); + } + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_visible_tests_a_rows_slice_after_a_removal_under_its_head_answers_as_its_whole_chain: + * bugs row 134 (orev-33 M1): a removal of a version under a key's head, as a + * fold's drop or a prune takes one, moves the range's count, so a row's slice + * walked before it is walked again after it and answers as the key's whole + * chain does, the removed ender's bytes still on its page. + */ +void +test_txn_visible_tests_a_rows_slice_after_a_removal_under_its_head_answers_as_its_whole_chain(void) +{ + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held; + HostSharedRecord state; + HostXid writer; + HostXid ender; + HostXid locker; + SimcTaken mvcc; + HostSnapshot snapshots[1]; + HostDeltaHold hold; + StdVec whole; + StdVec removed; + TxnMemtableVersionHandle gone; + TxnMemtableSliceSlice slice; + StdVec expected; + StdVec got; + size_t at = 0; + bool has_at = false; + TxnMemtableOp lock = {.tag = TXN_MEMTABLE_OP_LOCK, .u.lock = TXN_MEMTABLE_LOCK_MODE_SHARE}; + HostError error; + + simc_end(false); + held = txn_visible_tests_made(io, 9700); + state = host_shared_Held_record(&held); + writer = simc_begin(); + ender = simc_begin(); + locker = simc_begin(); + txn_visible_tests_written(io, state, "k", txn_visible_tests_by(writer, 0, 0), 1); + simc_commit(writer); + txn_visible_tests_written(io, state, "k", txn_visible_tests_by(ender, 0, 0), 0); + simc_commit(ender); + txn_visible_tests_pushing(io, state, "k", txn_visible_tests_by(locker, 0, 0), lock); + mvcc = simc_snapshot(HOST_SNAPSHOT_KIND_MVCC, false); + snapshots[0] = simc_Taken_lent(&mvcc); + hold = txn_visible_tests_hold(io); + for (int walk = 0; walk < 2; walk++) + { + slice = txn_visible_tests_slice(io, &hold, state, 1); + txn_memtable_slice_Slice_drop(&slice); + } + whole = txn_visible_tests_versioned(io, &hold, state, "k"); + gone = STD_VEC_AT(&whole, TxnMemtableVersionVersion, 1).handle; + txn_memtable_version_versions_drop(&whole, 0); + if (!txn_memtable_table_removed(&TXN_MEMTABLE_TABLE_ROWS_TREE_OPS, io, state, (const uint8 *) "k", 1, &gone, 1, + &removed, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + STD_ASSERT(__LINE__, 9, removed.len == 1 && STD_VEC_AT(&removed, bool, 0), + "table::removed::(io, state, b\"k\", &[gone]).unwrap() == [true]"); + std_vec_free_Checked(&removed, sizeof(bool), _Alignof(bool)); + whole = txn_visible_tests_versioned(io, &hold, state, "k"); + for (size_t k = 0; !has_at && k < whole.len; k++) + { + TxnVisibleRowVersion row; + + has_at = txn_visible_RowVersion_of(&STD_VEC_AT(&whole, TxnMemtableVersionVersion, k), &row); + at = k; + } + expected = txn_visible_tests_answers(io, snapshots, lengthof(snapshots), &whole, has_at, at, 1); + slice = txn_visible_tests_slice(io, &hold, state, 1); + got = txn_visible_tests_answers(io, snapshots, lengthof(snapshots), &slice.versions, slice.has_row, slice.row, 1); + STD_ASSERT_EQ(__LINE__, 9, txn_visible_tests_strings_eq(&got, &expected), &got, txn_visible_tests_strings_Debug_fmt, + &expected, txn_visible_tests_strings_Debug_fmt); + txn_visible_tests_strings_drop(&got); + txn_visible_tests_strings_drop(&expected); + txn_memtable_slice_Slice_drop(&slice); + txn_memtable_version_versions_drop(&whole, 0); + host_delta_Hold_drop(&hold); + simc_Taken_drop(&mvcc); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + +/* + * test_txn_visible_tests_a_versions_epoch_counts_the_transforms_written_before_it: + * STORAGE_DESIGN §II.10's epochs (PLAN's base field 4): a version is at its + * base's epoch and one more for each pending transform it was written after, + * one whose commit ends before its writer's, its own transaction's of an + * earlier command, and every one for a running writer. + */ +void +test_txn_visible_tests_a_versions_epoch_counts_the_transforms_written_before_it(void) +{ + static const char *const keys[] = {"early", "before", "after", "late", "running"}; + static const uint32 wanted[] = {3, 3, 4, 5, 5}; + SimcTestIo test_io = simc_io(); + HostIoIo *io = &test_io.io; + HostSharedHeld held = txn_visible_tests_made(io, 9351); + HostSharedRecord state = host_shared_Held_record(&held); + HostXid early = simc_begin(); + HostXid ddl = simc_begin(); + HostXid later_ddl = simc_begin(); + HostXid late = simc_begin(); + HostXid running = simc_begin(); + TxnMemtableVersionWriter writers[5]; + HostXid ended[4]; + HostLsn ends[] = {100, 200, 250, 300}; + TxnVisibleLive chain[2] = {0}; + HostDeltaHold hold; + uint32 epochs[5]; + StdTupleRef got; + StdTupleRef want; + + writers[0] = txn_visible_tests_by(early, 0, 0); + writers[1] = txn_visible_tests_by(ddl, 0, 1); + writers[2] = txn_visible_tests_by(ddl, 0, 3); + writers[3] = txn_visible_tests_by(late, 0, 0); + writers[4] = txn_visible_tests_by(running, 0, 0); + for (size_t k = 0; k < lengthof(keys); k++) + txn_visible_tests_written(io, state, keys[k], writers[k], k + 1); + ended[0] = early; + ended[1] = ddl; + ended[2] = later_ddl; + ended[3] = late; + for (size_t k = 0; k < lengthof(ended); k++) + { + TxnStatusOutcome outcome = {.tag = TXN_STATUS_OUTCOME_COMMITTED, .u.committed = ends[k]}; + TxnStatusRoomed roomed; + bool took; + HostError error; + + if (!txn_status_take(io, ended[k], outcome, &roomed, &took, &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + simc_commit(ended[k]); + } + chain[0].f0 = (TxnBasePending) + { + .xid = ddl.f0, .command = 2, .sub = ddl.f0, .transform = STD_VEC_NEW + }; + chain[0].has_f1 = true; + chain[0].f1 = 200; + chain[1].f0 = (TxnBasePending) + { + .xid = later_ddl.f0, .sub = later_ddl.f0, .transform = STD_VEC_NEW + }; + chain[1].has_f1 = true; + chain[1].f1 = 250; + hold = txn_visible_tests_hold(io); + for (size_t k = 0; k < lengthof(keys); k++) + { + StdVec versions = txn_visible_tests_versioned(io, &hold, state, keys[k]); + HostError error; + + if (!txn_visible_epoch(io, 3, chain, lengthof(chain), &STD_VEC_AT(&versions, TxnMemtableVersionVersion, 0), + &epochs[k], &error)) + STD_RESULT_EXPECT_FAILED(__LINE__, 0, "called `Result::unwrap()` on an `Err` value", &error, + host_Error_Debug_fmt); + txn_memtable_version_versions_drop(&versions, 0); + } + got = STD_TUPLE(5, STD_ITEM(&epochs[0], std_u32_Debug_fmt), STD_ITEM(&epochs[1], std_u32_Debug_fmt), + STD_ITEM(&epochs[2], std_u32_Debug_fmt), STD_ITEM(&epochs[3], std_u32_Debug_fmt), + STD_ITEM(&epochs[4], std_u32_Debug_fmt)); + want = STD_TUPLE(5, STD_ITEM(&wanted[0], std_u32_Debug_fmt), STD_ITEM(&wanted[1], std_u32_Debug_fmt), + STD_ITEM(&wanted[2], std_u32_Debug_fmt), STD_ITEM(&wanted[3], std_u32_Debug_fmt), + STD_ITEM(&wanted[4], std_u32_Debug_fmt)); + STD_ASSERT_EQ(__LINE__, 9, memcmp(epochs, wanted, sizeof(epochs)) == 0, &got, std_tuple_Debug_fmt, &want, + std_tuple_Debug_fmt); + host_delta_Hold_drop(&hold); + host_shared_Held_drop(&held); + simc_TestIo_drop(&test_io); +} + #endif /* DOLTGRES_TEST_TXN */ diff --git a/src/doltgres/txn/c/written.c b/src/doltgres/txn/c/written.c index 94976b4035aed..dd6b81a89480e 100644 --- a/src/doltgres/txn/c/written.c +++ b/src/doltgres/txn/c/written.c @@ -366,44 +366,42 @@ typedef struct txn_written_touched_t uint64 bytes; } txn_written_touched_t; -/* txn_written_touched_call adds the bytes to the tree's count and records the key unless it was just recorded or the list is full. */ +/* txn_written_touched_call adds the bytes to the tree's count and records the key unless it is a free space map's, was just recorded, or the list is full. */ static bool txn_written_touched_call(void *captures, HostIoIo *io, TxnWrittenWritten *written, uint64 *result, HostError *error) { txn_written_touched_t *c = captures; TxnWrittenTouched touched = {.storage = *c->storage, .of = c->of}; uint64 count = 0; + bool found = false; (void) io; - if (c->of != TXN_WRITTEN_OF_IDENTITY) + for (size_t k = 0; k < written->bytes.len && !found; k++) { - bool found = false; + txn_written_Bytes_t *at = &STD_VEC_AT(&written->bytes, txn_written_Bytes_t, k); - for (size_t k = 0; k < written->bytes.len && !found; k++) + if (host_Storage_PartialEq_eq(&at->f0, c->storage) && at->f1 == c->of) { - txn_written_Bytes_t *at = &STD_VEC_AT(&written->bytes, txn_written_Bytes_t, k); - - if (host_Storage_PartialEq_eq(&at->f0, c->storage) && at->f1 == c->of) - { - at->f2 = STD_ADD(u64, at->f2, c->bytes, 140, 17); - count = at->f2; - found = true; - } + at->f2 = STD_ADD(u64, at->f2, c->bytes, 140, 17); + count = at->f2; + found = true; } - if (!found) - { - txn_written_Bytes_t entry = {*c->storage, c->of, c->bytes}; + } + if (!found) + { + txn_written_Bytes_t entry = {*c->storage, c->of, c->bytes}; - if (!std_vec_try_reserve(&written->bytes, 1, sizeof(entry), _Alignof(txn_written_Bytes_t))) - { - *error = host_Error_out_of_memory(); - return false; - } - std_vec_push_Checked(&written->bytes, &entry, sizeof(entry), _Alignof(txn_written_Bytes_t)); - count = c->bytes; + if (!std_vec_try_reserve(&written->bytes, 1, sizeof(entry), _Alignof(txn_written_Bytes_t))) + { + *error = host_Error_out_of_memory(); + return false; } + std_vec_push_Checked(&written->bytes, &entry, sizeof(entry), _Alignof(txn_written_Bytes_t)); + count = c->bytes; } *result = count; + if (c->of == TXN_WRITTEN_OF_FREE) + return true; if (written->touched.len > 0) { const TxnWrittenTouched *last = &STD_VEC_AT(&written->touched, TxnWrittenTouched, written->touched.len - 1); @@ -434,8 +432,8 @@ txn_written_touched_call(void *captures, HostIoIo *io, TxnWrittenWritten *writte * version of a count of bytes, making the transaction's list at its first, * and adds the bytes to the tree's count with the key's, answering it; a key * just recorded isn't recorded twice, and a list at its cap is marked - * overflowed instead. A key of a database's identity tree adds no bytes, - * since that tree lives in the working state's delta and never spills. + * overflowed instead. A free space map's key adds its bytes and isn't + * recorded, since its take isn't taken back. */ bool txn_written_touched(HostIoIo *io, uint64 top, const HostStorage *storage, TxnWrittenOf of, const uint8 *key, @@ -445,8 +443,7 @@ txn_written_touched(HostIoIo *io, uint64 top, const HostStorage *storage, TxnWri .bytes = (uint64) STD_ADD(usize, version, key_len, 114, 17)}; HostOwnedHandle handle; - if (!txn_written_listing(io, top, &handle, error) || - (of != TXN_WRITTEN_OF_IDENTITY && !txn_spill_wrote(io, top, touched.bytes, error))) + if (!txn_written_listing(io, top, &handle, error) || !txn_spill_wrote(io, top, touched.bytes, error)) return false; return host_owned_TableTxnWrittenWritten_withU64(&TXN_WRITTEN_WRITTEN, io, handle, txn_written_touched_call, &touched, NULL, result, error); @@ -718,46 +715,72 @@ txn_written_ended(HostIoIo *io, HostError *error) return host_owned_TableTxnWrittenWritten_end(&TXN_WRITTEN_WRITTEN, io, handle, error); } -/* txn_written_pruned_one prunes one touched key in its tree, under the frame's held state or delta when its home is there. */ +/* + * txn_written_homed finds a touched key's home into held: its storage's state + * for a storage's tree, its database's working state's delta for its identity + * tree, or its repository's for its chunk tree; some is false for a home that + * is gone, or for a chunk tree of no repository, holding nothing to prune. + */ static bool -txn_written_pruned_one(HostIoIo *io, const TxnWrittenTouched *touched, bool has_aborting, uint64 aborting, - HostSharedHeld *held, bool *held_live, HostError *error) +txn_written_homed(HostIoIo *io, const TxnWrittenTouched *touched, HostSharedHeld *held, bool *some, + HostError *error) { - const TxnMemtableTableTreeOps *tree; uint32 repository; - bool some; if (touched->of == TXN_WRITTEN_OF_ROWS || touched->of == TXN_WRITTEN_OF_VALUES) + return txn_state_found(io, &touched->storage, some, held, error); + *some = false; + if (touched->of == TXN_WRITTEN_OF_IDENTITY) { - tree = touched->of == TXN_WRITTEN_OF_ROWS ? &TXN_MEMTABLE_TABLE_ROWS_TREE_OPS : - &TXN_MEMTABLE_MAP_VALUES_TREE_OPS; - if (!txn_state_found(io, &touched->storage, &some, held, error)) - return false; - } - else - { - tree = touched->of == TXN_WRITTEN_OF_IDENTITY ? &TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS : - &TXN_MEMTABLE_BRANCH_CHUNKS_TREE_OPS; - if (touched->of == TXN_WRITTEN_OF_IDENTITY) - { - repository = touched->storage.database; - if (repository == 0) - return true; - } - else if (!txn_spill_chunks_repository(&touched->storage, &repository)) + repository = touched->storage.database; + if (repository == 0) return true; - if (!txn_memtable_branch_find(HOST_SHARED_IO_WITHIN(io), repository, &some, held, error)) - return false; } - if (!some) + else if (!txn_spill_chunks_repository(&touched->storage, &repository)) return true; - *held_live = true; - if (!txn_memtable_table_pruned(tree, io, host_shared_Held_record(held), touched->key.ptr, touched->key.len, - has_aborting, aborting, error)) - return false; - *held_live = false; - host_shared_Held_drop(held); - return true; + return txn_memtable_branch_find(HOST_SHARED_IO_WITHIN(io), repository, some, held, error); +} + +/* txn_written_pruned_one prunes one touched key in its tree, under its home's held state or delta. */ +static bool +txn_written_pruned_one(HostIoIo *io, const TxnWrittenTouched *touched, bool has_aborting, uint64 aborting, + const HostSharedHeld *held, HostError *error) +{ + const TxnMemtableTableTreeOps *tree; + + if (touched->of == TXN_WRITTEN_OF_ROWS) + tree = &TXN_MEMTABLE_TABLE_ROWS_TREE_OPS; + else if (touched->of == TXN_WRITTEN_OF_VALUES) + tree = &TXN_MEMTABLE_MAP_VALUES_TREE_OPS; + else if (touched->of == TXN_WRITTEN_OF_IDENTITY) + tree = &TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS; + else + tree = &TXN_MEMTABLE_BRANCH_CHUNKS_TREE_OPS; + return txn_memtable_table_pruned(tree, io, host_shared_Held_record(held), touched->key.ptr, touched->key.len, + has_aborting, aborting, error); +} + +/* txn_written_Homed_t is a touched key's place in the prune's order: its tree, its storage, and its place in the list. */ +typedef struct txn_written_Homed_t +{ + TxnWrittenOf of; + HostStorage storage; + size_t at; +} txn_written_Homed_t; + +/* txn_written_homed_order orders touched keys by their home, and within one by their place in the list. */ +static int +txn_written_homed_order(const void *a_, const void *b_) +{ + const txn_written_Homed_t *a = a_; + const txn_written_Homed_t *b = b_; + uint64 left[5] = {(uint64) a->of, a->storage.tablespace, a->storage.database, a->storage.relation, a->at}; + uint64 right[5] = {(uint64) b->of, b->storage.tablespace, b->storage.database, b->storage.relation, b->at}; + + for (int i = 0; i < 5; i++) + if (left[i] != right[i]) + return left[i] < right[i] ? -1 : 1; + return 0; } /* txn_written_taken_frame_0_t is the state of aborted and sub_aborted across their pad: the keys and storages taken from the list, and a held home. */ @@ -765,15 +788,19 @@ typedef struct txn_written_taken_frame_0_t { HostIoIo *io; uint32 sub; + /* end is an aborted transaction's abort record's end. */ + uint64 end; HostError *error; bool ok; txn_written_Taken_t taken; bool taken_live; HostSharedHeld held; bool held_live; + StdVec order; + bool order_live; } txn_written_taken_frame_0_t; -/* txn_written_taken_pad_0 is the pad of aborted and sub_aborted: the held home, then what they took from the list, while live. */ +/* txn_written_taken_pad_0 is the pad of aborted and sub_aborted: the held home, the prune's order, then what they took from the list, while live. */ static void txn_written_taken_pad_0(void *frame_) { @@ -784,6 +811,11 @@ txn_written_taken_pad_0(void *frame_) frame->held_live = false; host_shared_Held_drop(&frame->held); } + if (frame->order_live) + { + frame->order_live = false; + std_vec_free_Checked(&frame->order, sizeof(txn_written_Homed_t), _Alignof(txn_written_Homed_t)); + } if (frame->taken_live) { frame->taken_live = false; @@ -796,15 +828,48 @@ txn_written_taken_pad_0(void *frame_) * tree, in its database's identity tree, or in its repository's chunk tree, * a storage whose state is gone, or a database whose working state's delta * is, holding nothing to prune, an aborting top-level transaction's own - * versions taken out by its xid. + * versions taken out by its xid. It prunes the keys grouped by their home, + * each home's in the list's order, and finds each home once, so its finds go + * with the homes the list names, not its keys (bugs row 309). */ static bool txn_written_pruned(txn_written_taken_frame_0_t *frame, bool has_aborting, uint64 aborting) { + const TxnWrittenTouched *home = NULL; + bool some = false; + + std_vec_with_capacity_Checked(&frame->order, frame->taken.touched.len, sizeof(txn_written_Homed_t), + _Alignof(txn_written_Homed_t)); + frame->order_live = true; for (size_t k = 0; k < frame->taken.touched.len; k++) - if (!txn_written_pruned_one(frame->io, &STD_VEC_AT(&frame->taken.touched, TxnWrittenTouched, k), has_aborting, - aborting, &frame->held, &frame->held_live, frame->error)) + { + const TxnWrittenTouched *touched = &STD_VEC_AT(&frame->taken.touched, TxnWrittenTouched, k); + txn_written_Homed_t homed = {touched->of, touched->storage, k}; + + std_vec_push_Checked(&frame->order, &homed, sizeof(homed), _Alignof(txn_written_Homed_t)); + } + if (frame->order.len > 1) + qsort(frame->order.ptr, frame->order.len, sizeof(txn_written_Homed_t), txn_written_homed_order); + for (size_t k = 0; k < frame->order.len; k++) + { + const TxnWrittenTouched *touched = &STD_VEC_AT(&frame->taken.touched, TxnWrittenTouched, + STD_VEC_AT(&frame->order, txn_written_Homed_t, k).at); + + if (home == NULL || home->of != touched->of || !host_Storage_PartialEq_eq(&home->storage, &touched->storage)) + { + if (frame->held_live) + { + frame->held_live = false; + host_shared_Held_drop(&frame->held); + } + if (!txn_written_homed(frame->io, touched, &frame->held, &some, frame->error)) + return false; + frame->held_live = some; + home = touched; + } + if (some && !txn_written_pruned_one(frame->io, touched, has_aborting, aborting, &frame->held, frame->error)) return false; + } return true; } @@ -841,7 +906,7 @@ txn_written_aborted_body_0(void *frame_) return; frame->taken_live = true; frame->ok = txn_spill_aborted(frame->io, frame->taken.top, frame->taken.spills.ptr, frame->taken.spills.len, - frame->error) && + frame->end, frame->error) && txn_written_pruned(frame, true, frame->taken.top) && txn_spill_unwrote(frame->io, false, 0, frame->error) && host_owned_TableTxnWrittenWritten_end(&TXN_WRITTEN_WRITTEN, frame->io, handle, frame->error); txn_written_taken_pad_0(frame); @@ -851,12 +916,13 @@ txn_written_aborted_body_0(void *frame_) * txn_written_aborted takes back the versions of the backend's aborted * top-level transaction: it takes its spills' entries out of the storages * it spilled and clears its pending spill, whose files the next checkpoint - * unlinks, prunes every key its list holds, then ends the list. + * unlinks, but its free space map's, which it marks with its abort record's + * end; prunes every key its list holds; then ends the list. */ bool -txn_written_aborted(HostIoIo *io, HostError *error) +txn_written_aborted(HostIoIo *io, uint64 end, HostError *error) { - txn_written_taken_frame_0_t frame = {.io = io, .error = error}; + txn_written_taken_frame_0_t frame = {.io = io, .end = end, .error = error}; std_landing(&frame, txn_written_aborted_body_0, txn_written_taken_pad_0); return frame.ok; diff --git a/src/doltgres/verify/port-map.tsv b/src/doltgres/verify/port-map.tsv index 88e7dfae85e93..1fb53e9901a6e 100644 --- a/src/doltgres/verify/port-map.tsv +++ b/src/doltgres/verify/port-map.tsv @@ -687,6 +687,7 @@ drop_in_place:: - host_ScratchFile_drop drop - - ::fmt - host_Xid_Debug_fmt derived - - ::eq - host_Xid_PartialEq_eq derived - - ::eq - - derived - a fieldless enum's PartialEq: C's == on the tag, inlined at each direct use +::fmt - host_Command_Debug_fmt derived - - ::eq - - derived - a fieldless enum's PartialEq: C's == on the tag, inlined at each direct use host::Snapshot<'a>::made - host_Snapshot_made method - - host::Snapshot<'a>::of - host_Snapshot_of method - - @@ -772,6 +773,7 @@ host::disk_space - host_disk_space fn - - host::tree_bytes - host_tree_bytes fn - - host::nap - host_nap fn - - host::wal_insert - host_wal_insert fn - - +::fmt - host_NewCid_Debug_fmt derived - - host::wal_new_cid - host_wal_new_cid fn - - host::wal_flush - host_wal_flush fn - - host::fold_point - host_fold_point fn - - @@ -6088,6 +6090,156 @@ txn::insert::loaded - txn_insert_loaded fn - - txn::insert::loaded::{closure#0} - txn_insert_loaded closure-inlined - - txn::insert::kept - txn_insert_kept fn - - txn::insert::replayed - txn_insert_replayed fn - - +txn::insert::tests::taken test txn_insert_tests_taken fn - - +txn::insert::tests::stored test txn_insert_tests_stored fn - - +txn::insert::tests::row test txn_insert_tests_row fn - - +txn::insert::tests::inserted test txn_insert_tests_inserted fn - - +txn::insert::tests::inserted::{closure#0} test txn_insert_tests_inserted closure-inlined - - +txn::insert::tests::inserts_take_heaps_tuple_ids_by_offsets_and_by_bytes test test_txn_insert_tests_inserts_take_heaps_tuple_ids_by_offsets_and_by_bytes test - - +txn::insert::tests::a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room test test_txn_insert_tests_a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room test - - +txn::insert::tests::a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room::{closure#0} test txn_insert_tests_last_page_put closure - - +txn::insert::tests::a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room::{closure#1} test test_txn_insert_tests_a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room closure-inlined - - +txn::insert::tests::a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room::{closure#2} test test_txn_insert_tests_a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room closure-inlined - - +txn::insert::tests::logged test txn_insert_tests_logged fn - - +txn::insert::tests::logged::{closure#0} test txn_insert_tests_logged closure-inlined - - +txn::insert::tests::logged::{closure#0}::{closure#0} test txn_insert_tests_logged closure-inlined - - +txn::insert::tests::logged::{closure#0}::{closure#1} test txn_insert_tests_logged closure-inlined - - +txn::insert::tests::logged::{closure#0}::{closure#2} test txn_insert_tests_logged closure-inlined - - +txn::insert::tests::logged::{closure#0}::{closure#3} test txn_insert_tests_logged closure-inlined - - +txn::insert::tests::logged::{closure#0}::{closure#4} test txn_insert_tests_logged closure-inlined - - +txn::insert::tests::free_entries test txn_insert_tests_free_entries fn - - +txn::insert::tests::free_entries::{closure#0} test txn_insert_tests_free_entries closure-inlined - - +txn::insert::tests::replica test txn_insert_tests_replica fn - - +txn::insert::tests::replica::{closure#0} test txn_insert_tests_replica closure-inlined - - +txn::insert::tests::replica::{closure#1} test txn_insert_tests_replica closure-inlined - - +txn::insert::tests::redone test txn_insert_tests_redone fn - - +txn::insert::tests::redone::{closure#0} test txn_insert_tests_redone closure-inlined - - +txn::insert::tests::redone::{closure#1} test txn_insert_tests_redone closure-inlined - - +txn::insert::tests::redone::{closure#2} test txn_insert_tests_redone closure-inlined - - +txn::insert::tests::redo_leaves_the_free_space_map_and_the_counter_as_the_primarys test test_txn_insert_tests_redo_leaves_the_free_space_map_and_the_counter_as_the_primarys test - - +txn::insert::tests::redo_leaves_the_free_space_map_and_the_counter_as_the_primarys::{closure#0} test test_txn_insert_tests_redo_leaves_the_free_space_map_and_the_counter_as_the_primarys closure-inlined - - +txn::insert::tests::redo_leaves_the_free_space_map_and_the_counter_as_the_primarys::{closure#1} test test_txn_insert_tests_redo_leaves_the_free_space_map_and_the_counter_as_the_primarys closure-inlined - - +txn::insert::tests::a_take_redone_again_changes_nothing test test_txn_insert_tests_a_take_redone_again_changes_nothing test - - +txn::insert::tests::a_take_redone_again_changes_nothing::{closure#0} test test_txn_insert_tests_a_take_redone_again_changes_nothing closure-inlined - - +txn::insert::tests::an_insert_maps_its_out_of_line_values_in_its_window test test_txn_insert_tests_an_insert_maps_its_out_of_line_values_in_its_window test - - +txn::insert::tests::an_insert_maps_its_out_of_line_values_in_its_window::{closure#0} test test_txn_insert_tests_an_insert_maps_its_out_of_line_values_in_its_window closure-inlined - - +txn::insert::tests::an_insert_maps_its_out_of_line_values_in_its_window::{closure#1} test test_txn_insert_tests_an_insert_maps_its_out_of_line_values_in_its_window closure-inlined - - +txn::insert::tests::an_insert_maps_its_out_of_line_values_in_its_window::{closure#2} test test_txn_insert_tests_an_insert_maps_its_out_of_line_values_in_its_window closure-inlined - - +txn::insert::tests::an_insert_maps_its_out_of_line_values_in_its_window::{closure#3} test test_txn_insert_tests_an_insert_maps_its_out_of_line_values_in_its_window closure-inlined - - +txn::insert::tests::linked test txn_insert_tests_linked fn - - +txn::insert::tests::a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter test test_txn_insert_tests_a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter test - - +txn::insert::tests::a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter::{closure#0} test txn_insert_tests_gathering_inserter closure - - +txn::insert::tests::a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter::{closure#1} test txn_insert_tests_gathering_gatherer closure - - +txn::insert::tests::a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter::{closure#1}::{closure#0} test txn_insert_tests_gathering_between closure - - +txn::insert::tests::a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter::{closure#2} test test_txn_insert_tests_a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter closure-inlined - - +txn::insert::tests::an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary test test_txn_insert_tests_an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary test - - +txn::insert::tests::an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary::{closure#0} test test_txn_insert_tests_an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary closure-inlined - - +txn::insert::tests::an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary::{closure#1} test test_txn_insert_tests_an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary closure-inlined - - +txn::insert::tests::an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary::{closure#2} test test_txn_insert_tests_an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary closure-inlined - - +txn::insert::tests::mapped test txn_insert_tests_mapped fn - - +txn::insert::tests::mapped::{closure#0} test txn_insert_tests_mapped closure-inlined - - +txn::insert::tests::valued test txn_insert_tests_valued fn - - +txn::insert::tests::valued::{closure#0} test txn_insert_tests_valued closure-inlined - - +txn::insert::tests::a_row_whose_value_references_sit_in_an_overflow_chunk_redoes_its_map_entries test test_txn_insert_tests_a_row_whose_value_references_sit_in_an_overflow_chunk_redoes_its_map_entries test - - +txn::insert::tests::a_row_whose_value_references_sit_in_an_overflow_chunk_redoes_its_map_entries::{closure#0} test test_txn_insert_tests_a_row_whose_value_references_sit_in_an_overflow_chunk_redoes_its_map_entries closure-inlined - - +txn::insert::tests::written test txn_insert_tests_written fn - - +txn::insert::tests::written::{closure#0} test txn_insert_tests_written closure-inlined - - +txn::insert::tests::written::{closure#1} test txn_insert_tests_written closure-inlined - - +txn::insert::tests::written::{closure#2} test txn_insert_tests_unpacked closure - - +txn::insert::tests::an_insert_that_fails_after_its_chunks_maps_no_value test test_txn_insert_tests_an_insert_that_fails_after_its_chunks_maps_no_value test - - +txn::insert::tests::an_insert_that_fails_after_its_chunks_maps_no_value::{closure#0} test test_txn_insert_tests_an_insert_that_fails_after_its_chunks_maps_no_value closure-inlined - - +txn::insert::tests::an_insert_that_fails_after_its_chunks_maps_no_value::{closure#1} test test_txn_insert_tests_an_insert_that_fails_after_its_chunks_maps_no_value closure-inlined - - +txn::insert::tests::an_insert_that_fails_after_its_chunks_maps_no_value::{closure#2} test test_txn_insert_tests_an_insert_that_fails_after_its_chunks_maps_no_value closure-inlined - - +txn::insert::tests::redo_from_any_point_gives_a_row_with_all_its_map_entries_or_none test test_txn_insert_tests_redo_from_any_point_gives_a_row_with_all_its_map_entries_or_none test - - +txn::insert::tests::a_kept_values_count_add_is_redone_once test test_txn_insert_tests_a_kept_values_count_add_is_redone_once test - - +txn::insert::tests::a_kept_values_count_add_is_redone_once::{closure#0} test txn_insert_tests_counts closure - - +txn::insert::tests::a_kept_values_count_add_is_redone_once::{closure#0}::{closure#0} test txn_insert_tests_counts closure-inlined - - +txn::insert::tests::a_load_moves_the_counter_past_every_loaded_tid_and_left_page test test_txn_insert_tests_a_load_moves_the_counter_past_every_loaded_tid_and_left_page test - - +txn::insert::tests::a_load_moves_the_counter_past_every_loaded_tid_and_left_page::{closure#0} test txn_insert_tests_left_inside closure - - +txn::insert::tests::a_load_moves_the_counter_past_every_loaded_tid_and_left_page::{closure#0}::{closure#0} test txn_insert_tests_left_step closure - - +txn::insert::tests::a_load_moves_the_counter_past_every_loaded_tid_and_left_page::{closure#0}::{closure#1} test txn_insert_tests_left_then closure - - +txn::insert::tests::a_load_moves_the_counter_past_every_loaded_tid_and_left_page::{closure#1} test test_txn_insert_tests_a_load_moves_the_counter_past_every_loaded_tid_and_left_page closure-inlined - - +txn::insert::tests::a_load_moves_the_counter_past_every_loaded_tid_and_left_page::{closure#2} test test_txn_insert_tests_a_load_moves_the_counter_past_every_loaded_tid_and_left_page closure-inlined - - +txn::insert::tests::an_insert_logical_decoding_reads_logs_heaps_record_of_its_command test test_txn_insert_tests_an_insert_logical_decoding_reads_logs_heaps_record_of_its_command test - - +txn::insert::tests::an_insert_logical_decoding_reads_logs_heaps_record_of_its_command::{closure#0} test txn_insert_tests_ours closure - - +txn::insert::tests::an_insert_logical_decoding_reads_logs_heaps_record_of_its_command::{closure#0}::{closure#0} test txn_insert_tests_ours closure-inlined - - +txn::insert::tests::an_insert_logical_decoding_reads_logs_heaps_record_of_its_command::{closure#1} test txn_insert_tests_own closure - - +txn::insert::tests::an_insert_logical_decoding_reads_logs_heaps_record_of_its_command::{closure#1}::{closure#0} test txn_insert_tests_own closure-inlined - - +txn::insert::tests::an_insert_logical_decoding_reads_logs_heaps_record_of_its_command::{closure#2} test test_txn_insert_tests_an_insert_logical_decoding_reads_logs_heaps_record_of_its_command closure-inlined - - +txn::insert::tests::chains test txn_insert_tests_chains fn - - +txn::insert::tests::chains::{closure#0} test txn_insert_tests_chains_each closure - - +txn::insert::tests::a_checkpoints_runs_and_every_record_after_redo_as_the_primary test test_txn_insert_tests_a_checkpoints_runs_and_every_record_after_redo_as_the_primary test - - +txn::insert::tests::a_checkpoints_runs_and_every_record_after_redo_as_the_primary::{closure#0} test test_txn_insert_tests_a_checkpoints_runs_and_every_record_after_redo_as_the_primary closure-inlined - - +txn::insert::tests::a_checkpoints_runs_and_every_record_after_redo_as_the_primary::{closure#1} test test_txn_insert_tests_a_checkpoints_runs_and_every_record_after_redo_as_the_primary closure-inlined - - +txn::insert::tests::a_checkpoints_runs_and_every_record_after_redo_as_the_primary::{closure#2} test txn_insert_tests_no_edits closure - - +txn::insert::tests::a_checkpoints_runs_and_every_record_after_redo_as_the_primary::{closure#3} test txn_insert_tests_no_marks closure - - +txn::insert::tests::a_checkpoints_runs_and_every_record_after_redo_as_the_primary::{closure#4} test test_txn_insert_tests_a_checkpoints_runs_and_every_record_after_redo_as_the_primary closure-inlined - - +txn::insert::tests::straddled test txn_insert_tests_straddled fn - - +txn::insert::tests::straddled::{closure#0} test txn_insert_tests_checkpointer closure - - +txn::insert::tests::straddled::{closure#1} test txn_insert_tests_straddle_inserter closure - - +txn::insert::tests::straddled::{closure#1}::{closure#0} test txn_insert_tests_straddle_inserter closure-inlined - - +txn::insert::tests::straddled::{closure#2} test txn_insert_tests_straddled closure-inlined - - +txn::insert::tests::straddled::{closure#2}::{closure#0} test txn_insert_tests_straddled closure-inlined - - +txn::insert::tests::straddled::{closure#3} test txn_insert_tests_straddled closure-inlined - - +txn::insert::tests::a_window_between_a_sweeps_rows_and_its_cut_leaves_its_take_without_its_row test test_txn_insert_tests_a_window_between_a_sweeps_rows_and_its_cut_leaves_its_take_without_its_row test - - +txn::insert::tests::windows_after_a_sweeps_cut_are_in_its_runs_with_neither_their_rows_nor_their_takes test test_txn_insert_tests_windows_after_a_sweeps_cut_are_in_its_runs_with_neither_their_rows_nor_their_takes test - - +txn::insert::tests::an_aborted_inserts_value_map_versions_go_at_its_take_back_and_its_free_space_take_stays test test_txn_insert_tests_an_aborted_inserts_value_map_versions_go_at_its_take_back_and_its_free_space_take_stays test - - +txn::insert::tests::an_aborted_inserts_value_map_versions_go_at_its_take_back_and_its_free_space_take_stays::{closure#0} test txn_insert_tests_value_entry closure - - +txn::insert::tests::ops test txn_insert_tests_ops fn - - +txn::insert::tests::ops::{closure#0} test txn_insert_tests_named closure - - +txn::insert::tests::ops::{closure#1} test txn_insert_tests_ops closure-inlined - - +txn::insert::tests::updated test txn_insert_tests_updated fn - - +txn::insert::tests::updated::{closure#0} test txn_insert_tests_updated_decide closure - - +txn::insert::tests::updated::{closure#0}::{closure#0} test txn_insert_tests_updated_decide closure-inlined - - +txn::insert::tests::replayed_rows test txn_insert_tests_replayed_rows fn - - +txn::insert::tests::an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary test test_txn_insert_tests_an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary test - - +txn::insert::tests::an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary::{closure#0} test txn_insert_tests_update_made closure - - +txn::insert::tests::an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary::{closure#1} test txn_insert_tests_key closure - - +txn::insert::tests::an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary::{closure#2} test test_txn_insert_tests_an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary closure-inlined - - +txn::insert::tests::a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids test test_txn_insert_tests_a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids test - - +txn::insert::tests::a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids::{closure#0} test txn_insert_tests_made_in closure - - +txn::insert::tests::a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids::{closure#1} test txn_insert_tests_key closure - - +txn::insert::tests::a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids::{closure#2} test txn_insert_tests_copy closure - - +txn::insert::tests::a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids::{closure#3} test txn_insert_tests_rowed closure - - +txn::insert::tests::an_updates_combo_command_shows_on_the_row_it_ends test test_txn_insert_tests_an_updates_combo_command_shows_on_the_row_it_ends test - - +txn::insert::tests::an_updates_combo_command_shows_on_the_row_it_ends::{closure#0} test txn_insert_tests_key closure - - +txn::insert::tests::an_updates_combo_command_shows_on_the_row_it_ends::{closure#1} test txn_insert_tests_shows closure - - +txn::insert::tests::an_updates_combo_command_shows_on_the_row_it_ends::{closure#1}::{closure#0} test txn_insert_tests_shows closure-inlined - - +txn::insert::tests::a_speculative_insert_holds_its_token_until_confirmed test test_txn_insert_tests_a_speculative_insert_holds_its_token_until_confirmed test - - +txn::insert::tests::a_speculative_insert_holds_its_token_until_confirmed::{closure#0} test txn_insert_tests_completion closure - - +txn::insert::tests::a_speculative_insert_holds_its_token_until_confirmed::{closure#1} test test_txn_insert_tests_a_speculative_insert_holds_its_token_until_confirmed closure-inlined - - +txn::insert::tests::an_aborted_updates_new_row_and_end_go_at_the_take_back test test_txn_insert_tests_an_aborted_updates_new_row_and_end_go_at_the_take_back test - - +txn::insert::tests::an_aborted_updates_new_row_and_end_go_at_the_take_back::{closure#0} test txn_insert_tests_key closure - - +::equal test txn_insert_tests_Folded_Equal_equal method - - +txn::insert::tests::a_resolve_whose_stripe_moves_three_times_holds_its_writers_back test test_txn_insert_tests_a_resolve_whose_stripe_moves_three_times_holds_its_writers_back test - - +txn::insert::tests::a_resolve_whose_stripe_moves_three_times_holds_its_writers_back::{closure#0} test txn_insert_tests_room closure - - +txn::insert::tests::a_resolve_whose_stripe_moves_three_times_holds_its_writers_back::{closure#1} test txn_insert_tests_resolver closure - - +txn::insert::tests::a_resolve_whose_stripe_moves_three_times_holds_its_writers_back::{closure#2} test txn_insert_tests_pusher closure - - +txn::insert::tests::a_resolve_whose_stripe_moves_three_times_holds_its_writers_back::{closure#3} test test_txn_insert_tests_a_resolve_whose_stripe_moves_three_times_holds_its_writers_back closure-inlined - - +txn::insert::tests::kept_across test txn_insert_tests_kept_across fn - - +txn::insert::tests::kept_across::{closure#0} test txn_insert_tests_kept_across closure-inlined - - +txn::insert::tests::kept_across::{closure#1} test txn_insert_tests_checkpointer closure - - +txn::insert::tests::kept_across::{closure#2} test txn_insert_tests_kept_updater closure - - +txn::insert::tests::kept_across::{closure#2}::{closure#0} test txn_insert_tests_go closure - - +txn::insert::tests::kept_across::{closure#3} test txn_insert_tests_kept_across closure-inlined - - +txn::insert::tests::kept_across::{closure#4} test txn_insert_tests_counts closure - - +txn::insert::tests::a_checkpoint_between_an_updates_count_adds_and_its_row_leaves_the_counts_once test test_txn_insert_tests_a_checkpoint_between_an_updates_count_adds_and_its_row_leaves_the_counts_once test - - +txn::insert::tests::a_checkpoint_between_an_updates_count_adds_and_its_row_leaves_the_counts_once::{closure#0} test txn_insert_tests_kept_between closure - - +txn::insert::tests::a_checkpoint_before_or_after_an_update_leaves_its_kept_counts_once test test_txn_insert_tests_a_checkpoint_before_or_after_an_update_leaves_its_kept_counts_once test - - +txn::insert::tests::a_checkpoint_before_or_after_an_update_leaves_its_kept_counts_once::{closure#0} test txn_insert_tests_kept_before closure - - +txn::insert::tests::a_checkpoint_before_or_after_an_update_leaves_its_kept_counts_once::{closure#1} test txn_insert_tests_kept_after closure - - +txn::insert::tests::a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both test test_txn_insert_tests_a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both test - - +txn::insert::tests::a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both::{closure#0} test test_txn_insert_tests_a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both closure-inlined - - +txn::insert::tests::a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both::{closure#1} test txn_insert_tests_checkpointer closure - - +txn::insert::tests::a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both::{closure#2} test txn_insert_tests_moving_updater closure - - +txn::insert::tests::a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both::{closure#2}::{closure#0} test txn_insert_tests_go closure - - +txn::insert::tests::a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both::{closure#3} test test_txn_insert_tests_a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both closure-inlined - - +txn::insert::tests::resolving test txn_insert_tests_resolving fn - - +txn::insert::tests::resolving::{closure#0} test txn_insert_tests_resolving_resolver closure - - +txn::insert::tests::a_resolve_whose_first_target_is_refused_ends_in_bounded_turns test test_txn_insert_tests_a_resolve_whose_first_target_is_refused_ends_in_bounded_turns test - - +txn::insert::tests::a_resolve_whose_first_target_is_refused_ends_in_bounded_turns::{closure#0} test txn_insert_tests_returned_is closure - - +txn::insert::tests::a_resolve_whose_first_target_is_refused_ends_in_bounded_turns::{closure#0}::{closure#0} test txn_insert_tests_returned_is closure-inlined - - +txn::insert::tests::a_frozen_inserts_row_change_replays_with_no_status_room test test_txn_insert_tests_a_frozen_inserts_row_change_replays_with_no_status_room test - - txn::memtable::branch::node_key - txn_memtable_branch_node_key fn - - txn::memtable::branch::piece_key - txn_memtable_branch_piece_key fn - - txn::memtable::branch::first_piece - txn_memtable_branch_first_piece fn - - @@ -6316,6 +6468,7 @@ txn::memtable::map::tests::a_cut_reads_the_counter_with_its_blocks_entry_from_th txn::memtable::map::tests::a_cut_reads_the_counter_with_its_blocks_entry_from_the_last_range::{closure#0} test txn_memtable_map_tests_values closure - - txn::memtable::map::tests::a_cut_reads_the_counter_with_its_blocks_entry_from_the_last_range::{closure#0}::{closure#0} test txn_memtable_map_tests_valued closure - - txn::memtable::map::tests::a_map_only_prepare_takes_its_entry_and_makes_no_rows_memtable test test_txn_memtable_map_tests_a_map_only_prepare_takes_its_entry_and_makes_no_rows_memtable test - - +txn::memtable::map::tests::a_take_whose_read_moved_answers_moved_and_pushes_nothing test test_txn_memtable_map_tests_a_take_whose_read_moved_answers_moved_and_pushes_nothing test - - txn::memtable::map::tests::a_map_only_prepare_takes_its_entry_and_makes_no_rows_memtable::{closure#0} test txn_memtable_map_tests_map_only_inside closure - - txn::memtable::map::tests::a_map_only_prepare_takes_its_entry_and_makes_no_rows_memtable::{closure#0}::{closure#0} test txn_memtable_map_tests_left closure - - txn::memtable::map::tests::a_map_only_prepare_takes_its_entry_and_makes_no_rows_memtable::{closure#0}::{closure#1} test txn_memtable_map_tests_nothing closure - - @@ -8162,6 +8315,78 @@ txn::vacuum::taken_out::{closure#0} - txn_vacuum_taken_out closure-inlined - - txn::vacuum::taken_out::{closure#1} - txn_vacuum_taken_out closure-inlined - - txn::vacuum::taken_out::{closure#2} - txn_vacuum_taken_out closure-inlined - - txn::vacuum::taken_out::{closure#2}::{closure#0} - txn_vacuum_taken_out closure-inlined - - +txn::vacuum::tests::every test txn_vacuum_tests_every fn - - +txn::vacuum::tests::with test txn_vacuum_tests_with fn - - +txn::vacuum::tests::inserted test txn_vacuum_tests_inserted fn - - +txn::vacuum::tests::ended test txn_vacuum_tests_ended fn - - +txn::vacuum::tests::entry test txn_vacuum_tests_entry fn - - +txn::vacuum::tests::logged test txn_vacuum_tests_logged fn - - +txn::vacuum::tests::logged::{closure#0} test txn_vacuum_tests_logged closure-inlined - - +txn::vacuum::tests::logged::{closure#0}::{closure#0} test txn_vacuum_tests_logged closure-inlined - - +txn::vacuum::tests::logged::{closure#0}::{closure#1} test txn_vacuum_tests_logged closure-inlined - - +txn::vacuum::tests::logged::{closure#0}::{closure#2} test txn_vacuum_tests_logged closure-inlined - - +txn::vacuum::tests::logged::{closure#0}::{closure#3} test txn_vacuum_tests_logged closure-inlined - - +txn::vacuum::tests::an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again test test_txn_vacuum_tests_an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again test - - +txn::vacuum::tests::an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again::{closure#0} test txn_vacuum_tests_aborted_primary closure - - +txn::vacuum::tests::an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again::{closure#1} test test_txn_vacuum_tests_an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again closure-inlined - - +txn::vacuum::tests::an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again::{closure#2} test txn_vacuum_tests_aborted_replica closure - - +txn::vacuum::tests::a_take_that_empties_its_entry_is_freed_with_all_the_bytes_the_entry_held test test_txn_vacuum_tests_a_take_that_empties_its_entry_is_freed_with_all_the_bytes_the_entry_held test - - +txn::vacuum::tests::a_take_that_empties_its_entry_is_freed_with_all_the_bytes_the_entry_held::{closure#0} test txn_vacuum_tests_emptying closure - - +txn::vacuum::tests::a_take_that_empties_its_entry_is_freed_with_all_the_bytes_the_entry_held::{closure#0}::{closure#0} test txn_vacuum_tests_emptying closure-inlined - - +txn::vacuum::tests::a_take_no_row_names_is_freed_only_once_its_taker_has_ended test test_txn_vacuum_tests_a_take_no_row_names_is_freed_only_once_its_taker_has_ended test - - +txn::vacuum::tests::a_take_no_row_names_is_freed_only_once_its_taker_has_ended::{closure#0} test txn_vacuum_tests_unnamed closure - - +txn::vacuum::tests::an_aborted_leaves_old_offsets_and_its_new_tuple_id_are_handed_out_again test test_txn_vacuum_tests_an_aborted_leaves_old_offsets_and_its_new_tuple_id_are_handed_out_again test - - +txn::vacuum::tests::an_aborted_leaves_old_offsets_and_its_new_tuple_id_are_handed_out_again::{closure#0} test txn_vacuum_tests_leaving closure - - +txn::vacuum::tests::an_aborted_leaves_old_offsets_and_its_new_tuple_id_are_handed_out_again::{closure#0}::{closure#0} test txn_vacuum_tests_leaving_insert closure - - +txn::vacuum::tests::a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone test test_txn_vacuum_tests_a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone test - - +txn::vacuum::tests::a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone::{closure#0} test txn_vacuum_tests_skipped_primary closure - - +txn::vacuum::tests::a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone::{closure#1} test txn_vacuum_tests_frees_is closure-inlined - - +txn::vacuum::tests::a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone::{closure#2} test txn_vacuum_tests_frees_is closure-inlined - - +txn::vacuum::tests::a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone::{closure#3} test txn_vacuum_tests_skipped_replica closure - - +txn::vacuum::tests::a_dead_tuple_id_on_a_full_block_keeps_its_bytes_until_its_free test test_txn_vacuum_tests_a_dead_tuple_id_on_a_full_block_keeps_its_bytes_until_its_free test - - +txn::vacuum::tests::a_dead_tuple_id_on_a_full_block_keeps_its_bytes_until_its_free::{closure#0} test txn_vacuum_tests_full_block closure - - +txn::vacuum::tests::updated test txn_vacuum_tests_updated fn - - +txn::vacuum::tests::updated::{closure#0} test txn_vacuum_tests_updated_decide closure - - +txn::vacuum::tests::updated::{closure#0}::{closure#0} test txn_vacuum_tests_updated_decide closure-inlined - - +txn::vacuum::tests::filled test txn_vacuum_tests_filled fn - - +txn::vacuum::tests::a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block test test_txn_vacuum_tests_a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block test - - +txn::vacuum::tests::a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block::{closure#0} test txn_vacuum_tests_pruning_primary closure - - +txn::vacuum::tests::a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block::{closure#0}::{closure#0} test txn_vacuum_tests_frees_is closure-inlined - - +txn::vacuum::tests::a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block::{closure#1} test txn_vacuum_tests_pruning_replica closure - - +txn::vacuum::tests::a_take_leaves_a_version_an_open_snapshot_may_see test test_txn_vacuum_tests_a_take_leaves_a_version_an_open_snapshot_may_see test - - +txn::vacuum::tests::a_take_leaves_a_version_an_open_snapshot_may_see::{closure#0} test txn_vacuum_tests_held_back closure - - +txn::vacuum::tests::a_dead_tuple_id_is_claimed_once test test_txn_vacuum_tests_a_dead_tuple_id_is_claimed_once test - - +txn::vacuum::tests::a_dead_tuple_id_is_claimed_once::{closure#0} test txn_vacuum_tests_claimed_once closure - - +txn::vacuum::tests::a_refused_markers_redo_keeps_the_version_as_the_primary_does test test_txn_vacuum_tests_a_refused_markers_redo_keeps_the_version_as_the_primary_does test - - +txn::vacuum::tests::a_refused_markers_redo_keeps_the_version_as_the_primary_does::{closure#0} test txn_vacuum_tests_refused_holds closure - - +txn::vacuum::tests::a_refused_markers_redo_keeps_the_version_as_the_primary_does::{closure#0}::{closure#0} test txn_vacuum_tests_refused_holds closure-inlined - - +txn::vacuum::tests::a_refused_markers_redo_keeps_the_version_as_the_primary_does::{closure#0}::{closure#0}::{closure#0} test txn_vacuum_tests_refused_holds closure-inlined - - +txn::vacuum::tests::a_refused_markers_redo_keeps_the_version_as_the_primary_does::{closure#1} test txn_vacuum_tests_refused_primary closure - - +txn::vacuum::tests::a_refused_markers_redo_keeps_the_version_as_the_primary_does::{closure#2} test txn_vacuum_tests_refused_replica closure - - +txn::vacuum::tests::every_raise_of_a_blocks_room_lowers_its_categorys_hint test test_txn_vacuum_tests_every_raise_of_a_blocks_room_lowers_its_categorys_hint test - - +txn::vacuum::tests::every_raise_of_a_blocks_room_lowers_its_categorys_hint::{closure#0} test txn_vacuum_tests_passed closure - - +txn::vacuum::tests::every_raise_of_a_blocks_room_lowers_its_categorys_hint::{closure#1} test txn_vacuum_tests_raised closure - - +txn::vacuum::tests::every_raise_of_a_blocks_room_lowers_its_categorys_hint::{closure#2} test txn_vacuum_tests_raised_dead closure - - +txn::vacuum::tests::every_raise_of_a_blocks_room_lowers_its_categorys_hint::{closure#3} test txn_vacuum_tests_raised_redo closure - - +txn::vacuum::tests::committed test txn_vacuum_tests_committed fn - - +txn::vacuum::tests::a_folded_rows_tuple_id_is_not_freed test test_txn_vacuum_tests_a_folded_rows_tuple_id_is_not_freed test - - +txn::vacuum::tests::a_fold_between_the_finds_reads_leaves_its_rows_held test test_txn_vacuum_tests_a_fold_between_the_finds_reads_leaves_its_rows_held test - - +txn::vacuum::tests::a_fold_between_the_finds_reads_leaves_its_rows_held::{closure#0} test txn_vacuum_tests_opened closure - - +txn::vacuum::tests::a_fold_between_the_finds_reads_leaves_its_rows_held::{closure#1} test txn_vacuum_tests_racing_vacuum closure - - +txn::vacuum::tests::a_fold_between_the_finds_reads_leaves_its_rows_held::{closure#2} test txn_vacuum_tests_racing_folder closure - - +txn::vacuum::tests::a_fold_between_the_finds_reads_leaves_its_rows_held::{closure#3} test test_txn_vacuum_tests_a_fold_between_the_finds_reads_leaves_its_rows_held closure-inlined - - +txn::vacuum::tests::deleted test txn_vacuum_tests_deleted fn - - +txn::vacuum::tests::a_dropped_rows_take_keeps_its_enders_xid_for_its_frees_horizon test test_txn_vacuum_tests_a_dropped_rows_take_keeps_its_enders_xid_for_its_frees_horizon test - - +txn::vacuum::tests::a_dropped_rows_take_keeps_its_enders_xid_for_its_frees_horizon::{closure#0} test txn_vacuum_tests_enders closure-inlined - - +txn::vacuum::tests::a_dropped_rows_take_keeps_its_enders_xid_for_its_frees_horizon::{closure#1} test txn_vacuum_tests_enders closure-inlined - - +txn::vacuum::tests::a_dropped_rows_take_keeps_its_enders_xid_for_its_frees_horizon::{closure#2} test txn_vacuum_tests_enders closure-inlined - - +txn::vacuum::tests::a_dropped_rows_take_keeps_its_enders_xid_for_its_frees_horizon::{closure#3} test txn_vacuum_tests_enders closure-inlined - - +txn::vacuum::tests::a_dropped_rows_take_keeps_its_enders_xid_for_its_frees_horizon::{closure#4} test txn_vacuum_tests_horizons closure-inlined - - +txn::vacuum::tests::a_takes_ender_widens_against_its_writer_across_an_epoch test test_txn_vacuum_tests_a_takes_ender_widens_against_its_writer_across_an_epoch test - - +txn::vacuum::tests::a_takes_ender_widens_against_its_writer_across_an_epoch::{closure#0} test txn_vacuum_tests_widened closure - - +txn::vacuum::tests::a_dropped_rows_tuple_id_waits_for_the_horizon_to_pass_its_ender test test_txn_vacuum_tests_a_dropped_rows_tuple_id_waits_for_the_horizon_to_pass_its_ender test - - +txn::vacuum::tests::a_dropped_rows_tuple_id_waits_for_the_horizon_to_pass_its_ender::{closure#0} test test_txn_vacuum_tests_a_dropped_rows_tuple_id_waits_for_the_horizon_to_pass_its_ender closure-inlined - - +txn::vacuum::tests::a_row_its_own_transaction_inserted_and_deleted_waits_for_the_horizon test test_txn_vacuum_tests_a_row_its_own_transaction_inserted_and_deleted_waits_for_the_horizon test - - ::fmt - txn_values_Entry_Debug_fmt derived - - ::eq - txn_values_Entry_PartialEq_eq derived - - txn::values::maxaligned - txn_values_maxaligned fn - - diff --git a/src/include/doltgres/chunks/packs.h b/src/include/doltgres/chunks/packs.h index be8b29a03c105..7a5b2f50f8401 100644 --- a/src/include/doltgres/chunks/packs.h +++ b/src/include/doltgres/chunks/packs.h @@ -168,6 +168,7 @@ extern void chunks_packs_table_packs_insert(StdVec *packs, uint64 number, const extern bool chunks_packs_table_packs_remove(StdVec *packs, uint64 number); extern ChunksPacksTail *chunks_packs_table_runs_get(const StdVec *runs, uint32 space); extern void chunks_packs_table_runs_insert(StdVec *runs, uint32 space, ChunksPacksTail tail); +extern void chunks_packs_table_runs_remove(StdVec *runs, uint32 space); extern void chunks_packs_Table_whole(const ChunksPacksTable *self, StdVec *result); extern void chunks_packs_Table_added(const ChunksPacksTableEntry *entry, ChunksPacksTail runs, StdVec *result); extern void chunks_packs_Table_reserved(uint64 bound, StdVec *result); diff --git a/src/include/doltgres/chunks/packs/backend.h b/src/include/doltgres/chunks/packs/backend.h index cfee5c36cd124..341dd28e0491a 100644 --- a/src/include/doltgres/chunks/packs/backend.h +++ b/src/include/doltgres/chunks/packs/backend.h @@ -78,7 +78,7 @@ extern bool chunks_packs_backend_copied_file(HostIoIoDoors io, const char *from_ const char *to_path, size_t to_path_len, ChunksErrorError *error); extern bool chunks_packs_backend_Packs_close(ChunksPacksBackendPacks *self, HostIoIo *io, const HostSharedReader *reading, ChunksErrorError *error); -extern bool chunks_packs_backend_Packs_flush(const ChunksPacksBackendPacks *self, HostIoIo *io, +extern bool chunks_packs_backend_Packs_flush(const ChunksPacksBackendPacks *self, HostIoTakesShort io, const HostSharedReader *reading, ChunksErrorError *error); extern bool chunks_packs_backend_Packs_copied(const ChunksPacksBackendPacks *self, HostIoIo *io, const HostSharedReader *reading, const char *into, size_t into_len, @@ -201,6 +201,9 @@ extern bool chunks_packs_backend_Packs_even_manifest(const ChunksPacksBackendPac ChunksErrorError *error); extern bool chunks_packs_backend_Packs_compact(const ChunksPacksBackendPacks *self, HostIoTakesShort io, const HostSharedReader *reading, bool all, ChunksErrorError *error); +extern bool chunks_packs_backend_Packs_remove_space(const ChunksPacksBackendPacks *self, HostIoTakesShort io, + const HostSharedReader *reading, uint32 space, + ChunksErrorError *error); #ifdef DOLTGRES_TEST_CHUNKS extern void test_chunks_packs_backend_tests_a_collection_keeps_every_pack_from_the_lowest_a_process_has_open(void); diff --git a/src/include/doltgres/chunks/packs/lists.h b/src/include/doltgres/chunks/packs/lists.h index 598c050b5bdf0..fd66946ba717e 100644 --- a/src/include/doltgres/chunks/packs/lists.h +++ b/src/include/doltgres/chunks/packs/lists.h @@ -90,6 +90,8 @@ extern bool chunks_packs_lists_Rewrite_begin(ChunksPacksViewView *view, HostShar HostSharedFlagIo *io, ChunksPacksListsChange *result, ChunksErrorError *error); extern bool chunks_packs_lists_Rewrite_compact(ChunksPacksListsChange *self, bool all, ChunksErrorError *error); +extern bool chunks_packs_lists_Rewrite_remove_space(ChunksPacksListsChange *self, uint32 space, + ChunksErrorError *error); #ifdef DOLTGRES_TEST_CHUNKS extern HostSharedOwned chunks_packs_lists_tests_repository(HostIoIo *io, const StdPathBuf *dir); @@ -105,6 +107,7 @@ extern void test_chunks_packs_lists_tests_only_a_list_file_past_its_slack_is_wri extern void test_chunks_packs_lists_tests_a_retirements_records_lead_every_view_to_the_copies(void); extern void test_chunks_packs_lists_tests_a_refresh_across_a_whole_write_lists_the_packs_before_it_or_after_it(void); extern void test_chunks_packs_lists_tests_a_whole_write_that_fails_before_its_renames_leaves_the_count_even(void); +extern void test_chunks_packs_lists_tests_a_removed_tablespace_reads_a_runs_file_made_again_from_its_start(void); #endif #endif /* DOLTGRES_CHUNKS_PACKS_LISTS_H */ diff --git a/src/include/doltgres/chunks/packs/view.h b/src/include/doltgres/chunks/packs/view.h index cb9ae7b5065c6..64c34d7a23808 100644 --- a/src/include/doltgres/chunks/packs/view.h +++ b/src/include/doltgres/chunks/packs/view.h @@ -175,7 +175,7 @@ extern void chunks_packs_view_View_Default_default(ChunksPacksViewView *result); extern void chunks_packs_view_View_new(uint32 repository, uint32 own, ChunksPacksViewView *result); extern void chunks_packs_view_View_drop(ChunksPacksViewView *self); extern void chunks_packs_view_View_space_dir(const ChunksPacksViewView *self, uint32 space, StdPathBuf *result); -extern bool chunks_packs_view_View_refresh(ChunksPacksViewView *self, HostIoIo *io, HostSharedRecord record, +extern bool chunks_packs_view_View_refresh(ChunksPacksViewView *self, HostIoTakesShort io, HostSharedRecord record, ChunksErrorError *error); extern bool chunks_packs_view_View_refresh_under(ChunksPacksViewView *self, HostSharedLongIo *io, HostSharedRecord record, ChunksPacksSharedSnapshot *result, @@ -194,6 +194,7 @@ extern const ChunksIndexAddressIndexLocator *chunks_packs_view_View_forwards(con extern const ChunksPacksTable *chunks_packs_view_View_table(const ChunksPacksViewView *self); extern ChunksPacksViewReadRuns *chunks_packs_view_runs_get(const StdVec *runs, uint32 space); extern ChunksPacksViewReadRuns *chunks_packs_view_runs_entry(StdVec *runs, uint32 space); +extern void chunks_packs_view_runs_remove(StdVec *runs, uint32 space); extern bool chunks_packs_view_missing_contains(const StdVec *missing, uint32 space); /* reads.c's reads through a view. */ diff --git a/src/include/doltgres/chunks/trailer.h b/src/include/doltgres/chunks/trailer.h index c20592bd5a73f..e66af5d3c4cdd 100644 --- a/src/include/doltgres/chunks/trailer.h +++ b/src/include/doltgres/chunks/trailer.h @@ -174,6 +174,8 @@ extern size_t chunks_trailer_Stamps_entries(const ChunksTrailerStamps *self); extern uint64 chunks_trailer_Stamps_entry_len(const ChunksTrailerStamps *self, size_t i); extern bool chunks_trailer_Stamps_first(const ChunksTrailerStamps *self, size_t i, ChunksTrailerStamp *result); extern void chunks_trailer_Stamps_entry(const ChunksTrailerStamps *self, size_t i, StdVec *result); +extern bool chunks_trailer_Stamps_stamp_of(const ChunksTrailerStamps *self, size_t i, uint64 tuple, + ChunksTrailerStamp *result); extern bool chunks_trailer_Stamps_holds(const ChunksTrailerStamps *self, size_t i, uint64 tuple); extern uint64 chunks_trailer_Stamps_max_tuple(const ChunksTrailerStamps *self); extern void chunks_trailer_Stamps_push(ChunksTrailerStamps *self, const ChunksTrailerStamp *entry, size_t entry_len); @@ -206,6 +208,7 @@ extern void test_chunks_trailer_tests_corrupt_runs_fail_without_overflowing_or_a extern void test_chunks_trailer_tests_a_flipped_byte_anywhere_in_a_trailer_fails_the_split(void); extern void test_chunks_trailer_tests_a_locator_keeps_its_three_numbers_at_their_widths(void); extern void test_chunks_trailer_tests_xxh3_digests_are_the_committed_table(void); +extern void test_chunks_trailer_tests_stamp_of_finds_each_copys_stamp_among_its_runs(void); extern void test_chunks_trailer_tests_a_leafs_value_ids_follow_its_stamps_and_refuse_every_other_form(void); extern void test_chunks_trailer_tests_internal_nodes_store_their_subtrees_highest_tuple_ids(void); #endif diff --git a/src/include/doltgres/doltgres/entries.h b/src/include/doltgres/doltgres/entries.h index dfd323ae57309..90944633bca35 100644 --- a/src/include/doltgres/doltgres/entries.h +++ b/src/include/doltgres/doltgres/entries.h @@ -37,6 +37,14 @@ #include "doltgres/txn/status.h" #include "doltgres/txn/values.h" +/* + * DOLTGRES_DROPPED_MARK is the file tblspc_redo leaves in a dropped + * tablespace's directory of repositories when its replay couldn't empty it, + * the collector's sweep's one sign to, since the worker connects to no + * database and pg_tablespace isn't nailed (bugs row 99). + */ +#define DOLTGRES_DROPPED_MARK "dropped" + /* doltgres_TupleUsizeAddressU64_t is a (usize, Address, u64): a value's field, its blob's root, and its length. */ typedef struct doltgres_TupleUsizeAddressU64_t { @@ -292,10 +300,29 @@ extern bool doltgres_entries_doltgres_index_blocks(const PgCatalogCaps *caps, co uint32 *blocks, bool *result, PgPgError *error); extern bool doltgres_entries_doltgres_relation_size_added(const PgCatalogCaps *caps, const PgRel *rel, const PgText *forkName, int64 size, int64 *result, PgPgError *error); +extern bool doltgres_entries_doltgres_visibility_counted(const PgCatalogCaps *caps, const PgRel *rel, uint32 *all_visible, + uint32 *all_frozen, bool *result, PgPgError *error); +extern bool doltgres_entries_doltgres_heap_only_refused(const PgCatalogCaps *caps, const PgRel *rel, bool *result, + PgPgError *error); +extern bool doltgres_entries_doltgres_visibility_kept(const PgCatalogCaps *caps, const PgRel *rel, bool *result, + PgPgError *error); +extern bool doltgres_entries_doltgres_visibility_bits(const PgCatalogCaps *caps, const PgRel *rel, uint32 blkno, + int32 mapbits, int32 *result, PgPgError *error); +extern bool doltgres_entries_doltgres_index_scan_io(const PgCatalogCaps *caps, uint32 indexoid, bool indexonly, + float8 tuples_fetched, float8 selectivity, float8 loop_count, + float8 random_page_cost, float8 seq_page_cost, + float8 cpu_operator_cost, float8 *max_io, float8 *min_io, + float8 *rand_heap_pages, PgPgError *error); +extern bool doltgres_entries_doltgres_bitmap_page_cost(const PgCatalogCaps *caps, uint32 relid, float8 tuples_fetched, + float8 pages_fetched, uint32 pages, float8 maxentries, + float8 random_page_cost, float8 cpu_operator_cost, + float8 *cost_per_page, PgPgError *error); extern bool doltgres_entries_doltgres_database_size(const PgCaps *caps, uint32 dbOid, int64 size, int64 *result, PgPgError *error); extern bool doltgres_entries_doltgres_tablespace_size(const PgCaps *caps, uint32 tblspcOid, int64 size, int64 *result, PgPgError *error); +extern bool doltgres_entries_doltgres_tablespace_emptied(const PgCaps *caps, uint32 tablespaceoid, bool redo, + PgPgError *error); extern bool doltgres_entries_doltgres_altering(const PgCatalogCaps *caps, const PgRel *relation, const PgAlterCmd *cmd, int32 rewrite, PgPgError *error); extern bool doltgres_entries_doltgres_column_dropped(const PgCatalogCaps *caps, uint32 relid, int32 attnum, diff --git a/src/include/doltgres/doltgres/entries/testing.h b/src/include/doltgres/doltgres/entries/testing.h index 6b5a38a13be46..976c85d5ca3eb 100644 --- a/src/include/doltgres/doltgres/entries/testing.h +++ b/src/include/doltgres/doltgres/entries/testing.h @@ -97,6 +97,8 @@ extern bool doltgres_entries_testing_doltgres_test_fold(const PgCatalogCaps *cap PgPgError *error); extern bool doltgres_entries_testing_doltgres_test_free_space(const PgCatalogCaps *caps, const PgRel *target, int32 block, PgDatum *result, PgPgError *error); +extern bool doltgres_entries_testing_doltgres_test_base_points(const PgCatalogCaps *caps, const PgRel *target, + PgDatum *result, PgPgError *error); extern bool doltgres_entries_testing_doltgres_test_base_sizes(const PgCatalogCaps *caps, const PgRel *target, PgDatum *result, PgPgError *error); extern bool doltgres_entries_testing_doltgres_test_index_entries(const PgCatalogCaps *caps, const PgRel *target, @@ -120,10 +122,15 @@ extern bool doltgres_entries_testing_doltgres_test_mapped(const PgCatalogCaps *c extern bool doltgres_entries_testing_doltgres_test_identity(const PgCatalogCaps *caps, int32 kind, uint32 number, int32 attnum, PgDatum *result, PgPgError *error); extern bool doltgres_entries_testing_doltgres_test_epochs(const PgCaps *caps, PgDatum *result, PgPgError *error); +extern bool doltgres_entries_testing_doltgres_test_identity_versions(const PgCaps *caps, int32 kind, uint32 number, + PgDatum *result, PgPgError *error); +extern bool doltgres_entries_testing_doltgres_test_save(const PgCaps *caps, PgDatum *result, PgPgError *error); extern bool doltgres_entries_testing_doltgres_test_base_levels(const PgCatalogCaps *caps, const PgRel *target, int32 *result, PgPgError *error); extern bool doltgres_entries_testing_doltgres_test_totals(const PgCatalogCaps *caps, const PgRel *target, PgDatum *result, PgPgError *error); +extern bool doltgres_entries_testing_doltgres_test_scan_shape(const PgCatalogCaps *caps, const PgRel *target, + PgDatum *result, PgPgError *error); extern bool doltgres_entries_testing_doltgres_test_count(const PgCatalogCaps *caps, const PgRel *target, PgDatum *result, PgPgError *error); diff --git a/src/include/doltgres/doltgres/entries_tail.h b/src/include/doltgres/doltgres/entries_tail.h index 9a6781e6ed599..75dbf9cc8b1da 100644 --- a/src/include/doltgres/doltgres/entries_tail.h +++ b/src/include/doltgres/doltgres/entries_tail.h @@ -107,6 +107,22 @@ typedef struct doltgres_entries_opened_t extern bool doltgres_entries_open(HostIoIo *io, doltgres_entries_opened_t *opened, PgPgError *error); extern void doltgres_entries_opened_drop(doltgres_entries_opened_t *opened); extern const RepoRepo *doltgres_entries_opened_repo(const doltgres_entries_opened_t *opened); + +/* + * DoltgresEntriesScanned is a scanned table's shape for its scans' costs: + * whether the fork stores it, whether the index is its primary key's, and + * the blocks of its tree and of its tuple id tree. + */ +typedef struct DoltgresEntriesScanned +{ + bool stored; + bool keyed; + TxnBaseBlockCounts rows; + TxnBaseBlockCounts tuple_ids; +} DoltgresEntriesScanned; + +extern bool doltgres_entries_scanned(const PgCatalogCaps *caps, uint32 indexoid, uint32 tableoid, + DoltgresEntriesScanned *result, PgPgError *error); extern void doltgres_entries_Record_drop(txn_TupleVecU8Op_t *record); /* DoltgresEntriesLockedOp is one operation on a backend's value under its IoMutex, with its captures. */ diff --git a/src/include/doltgres/doltgres/rewrite.h b/src/include/doltgres/doltgres/rewrite.h index 1e32fb24e4c31..d8b9645a292ac 100644 --- a/src/include/doltgres/doltgres/rewrite.h +++ b/src/include/doltgres/doltgres/rewrite.h @@ -195,16 +195,6 @@ typedef struct DoltgresRewriteRewritten TxnRewritePacked packed; } DoltgresRewriteRewritten; -/* - * DoltgresRewriteCarry is Carry: a version the new storage carries, its row - * change, the command it showed, and its row lock. - */ -typedef struct DoltgresRewriteCarry -{ - TxnRecordRowChange f0; - TxnMemtableVersionShown f1; - TxnMemtableLockMode f2; -} DoltgresRewriteCarry; extern void doltgres_rewrite_Read_drop(DoltgresRewriteRead *self); extern bool doltgres_rewrite_read(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, TxnBaseBase *base, uint64 horizon, DoltgresRewriteRead *result, HostError *error); diff --git a/src/include/doltgres/doltgres/scan.h b/src/include/doltgres/doltgres/scan.h index d152e8c751025..765cb80ef586b 100644 --- a/src/include/doltgres/doltgres/scan.h +++ b/src/include/doltgres/doltgres/scan.h @@ -29,6 +29,7 @@ #include "doltgres/host/shared.h" #include "doltgres/repo/lib.h" #include "doltgres/std/vec.h" +#include "doltgres/trees/cursor.h" #include "doltgres/trees/lib.h" #include "doltgres/trees/store.h" #include "doltgres/txn/base.h" @@ -187,8 +188,9 @@ typedef struct DoltgresScanLayout /* * DoltgresScanTrees is Trees: the trees of the base a read takes, its table - * tree and its tuple id tree, each none for a storage with no base, and its - * fold point. + * tree and its tuple id tree, each none for a storage with no base, its fold + * point, and the storage's base generation the base was read at or after, + * which a read checks after its delta read (txn_state_moved_past). */ typedef struct DoltgresScanTrees { @@ -197,6 +199,7 @@ typedef struct DoltgresScanTrees bool has_tuple_ids; TreesStoreTree tuple_ids; uint64 folded; + uint64 generation; } DoltgresScanTrees; /* @@ -270,9 +273,10 @@ extern bool doltgres_scan_bitmap_step(HostIoIo *io, HostOwnedHandle handle, Dolt extern bool doltgres_scan_bitmap_ended(HostIoIo *io, HostOwnedHandle handle, HostError *error); extern bool doltgres_scan_blocks(HostIoIo *io, HostOwnedHandle handle, uint64 *result, HostError *error); extern bool doltgres_scan_Direction_Debug_fmt(const void *value, struct StdFormatter *f); +extern DoltgresScanTrees doltgres_scan_Trees_of(const TxnBaseBase *base, uint64 generation); extern bool doltgres_scan_scan_begin(HostIoIo *io, TreesStoreNodeStore store, uint64 blocks, const TxnBaseBase *base, - StdVec *keyed, uint32 table, TreesKeyOrder *order, HostOwnedHandle *result, - HostError *error); + uint64 generation, StdVec *keyed, uint32 table, TreesKeyOrder *order, + HostOwnedHandle *result, HostError *error); extern bool doltgres_scan_next(HostIoIo *io, const DoltgresScanStores *stores, const DoltgresScanUnder *under, HostOwnedHandle handle, DoltgresScanDirection way, bool *has_result, DoltgresScanRow *result, HostError *error); @@ -284,12 +288,27 @@ extern bool doltgres_scan_seek(HostIoIo *io, const DoltgresScanStores *stores, H extern bool doltgres_scan_scan_end(HostIoIo *io, HostOwnedHandle handle, HostError *error); extern bool doltgres_scan_tid_valid(HostIoIo *io, HostOwnedHandle handle, uint64 tuple_id, uint16 most_offset, bool *result, HostError *error); + +/* + * DoltgresScanResolved is the memo of the base leaf a scan's fetches last + * resolved a row in, one immutable leaf named by its content at most, which + * lives no longer than its scan or its fetch: the leaf's row stamps, and a + * cursor kept in it, whose path starts at the root it was sought from. + */ +typedef struct DoltgresScanResolved +{ + TreesCursorLeafStamps stamps; + bool has_kept; + TreesCursorKeptCursor kept; +} DoltgresScanResolved; + +extern void doltgres_scan_Resolved_drop(DoltgresScanResolved *self); extern bool doltgres_scan_fetch(HostIoIo *io, TreesStoreNodeStore store, const DoltgresScanUnder *under, - DoltgresScanEpochs *epochs, DoltgresScanTrees trees, DoltgresScanLayout *layout, - uint64 tuple_id, bool *has_result, DoltgresScanRow *result, HostError *error); -extern bool doltgres_scan_keyed(HostIoIo *io, TreesStoreNodeStore store, HostSharedRecord state, bool has_tuple_ids, - TreesStoreTree tuple_ids, uint64 tuple_id, bool *has_result, StdVec *result_0, - bool *result_1, HostError *error); + DoltgresScanEpochs *epochs, DoltgresScanResolved *resolved, DoltgresScanTrees trees, + DoltgresScanLayout *layout, uint64 tuple_id, bool *has_result, DoltgresScanRow *result, + HostError *error); +extern bool doltgres_scan_keyed(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint64 tuple_id, + bool *has_result, StdVec *result_0, bool *result_1, HostError *error); extern bool doltgres_scan_based(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint64 tuple_id, bool *result, HostError *error); extern bool doltgres_scan_fetch_key(HostIoIo *io, TreesStoreNodeStore store, const DoltgresScanUnder *under, @@ -307,8 +326,9 @@ extern bool doltgres_scan_next_in_range(HostIoIo *io, TreesStoreNodeStore store, DoltgresScanRow *result, HostError *error); extern void doltgres_scan_splits(const StdVec *starts, size_t starts_len, StdVec *result); extern bool doltgres_scan_split_keys(const uint8 *bytes, size_t bytes_len, StdVec *result, HostError *error); -extern bool doltgres_scan_begin_parallel(HostIoIo *io, DoltgresScanShared *shared, StdVec *keyed, uint32 table, - TreesKeyOrder *order, HostOwnedHandle *result, HostError *error); +extern bool doltgres_scan_begin_parallel(HostIoIo *io, DoltgresScanShared *shared, uint64 generation, StdVec *keyed, + uint32 table, TreesKeyOrder *order, HostOwnedHandle *result, + HostError *error); extern bool doltgres_scan_is_parallel(HostIoIo *io, HostOwnedHandle handle, bool *result, HostError *error); extern bool doltgres_scan_start_unit(HostIoIo *io, HostOwnedHandle handle, uint64 unit, HostError *error); extern bool doltgres_scan_next_parallel(HostIoIo *io, const DoltgresScanStores *stores, @@ -344,6 +364,9 @@ extern void test_doltgres_scan_tests_a_leaders_split_keys_are_its_range_starts_s extern void test_doltgres_scan_tests_a_parallel_scan_over_an_empty_base_reads_each_delta_key_once_by_the_leaders_splits(void); extern void test_doltgres_scan_tests_a_sample_of_many_blocks_walks_the_delta_once(void); extern void test_doltgres_scan_tests_a_scan_with_no_sample_or_range_leaves_its_answers_as_they_were(void); +extern void test_doltgres_scan_tests_a_scan_moves_to_a_newer_base_at_its_next_read_and_gives_every_row_once(void); +extern void test_doltgres_scan_tests_a_scan_held_inside_its_delta_read_across_an_install_and_drop_gives_every_row_once(void); +extern void test_doltgres_scan_tests_fetches_through_one_memo_give_each_keyless_copys_own_stamp(void); #endif #endif /* DOLTGRES_DOLTGRES_SCAN_H */ diff --git a/src/include/doltgres/hooks.h b/src/include/doltgres/hooks.h index 325921306c889..563a2dc6e571a 100644 --- a/src/include/doltgres/hooks.h +++ b/src/include/doltgres/hooks.h @@ -310,6 +310,69 @@ extern int64 doltgres_database_size(Oid dbOid, int64 size); /* doltgres_tablespace_size adds the files of a tablespace's repositories, in calculate_tablespace_size. */ extern int64 doltgres_tablespace_size(Oid tblspcOid, int64 size); +/* + * doltgres_visibility_counted counts every block of a fork table all-visible + * and all-frozen, in visibilitymap_count, answering whether the fork stores + * the table. + */ +extern bool doltgres_visibility_counted(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen); + +/* + * doltgres_index_scan_io gives a fork table's part of an index scan's costs, + * in cost_index, in place of heap's page fetches: its I/O costs at either + * correlation and the leaves it reads at random. + */ +extern void doltgres_index_scan_io(Oid indexoid, bool indexonly, double tuples_fetched, double selectivity, + double loop_count, double random_page_cost, double seq_page_cost, + double cpu_operator_cost, double *max_io, double *min_io, double *rand_heap_pages); + +/* + * doltgres_bitmap_page_cost gives a fork table's cost per page fetched of a + * bitmap heap scan, in cost_bitmap_heap_scan, in place of heap's. + */ +extern void doltgres_bitmap_page_cost(Oid relid, double tuples_fetched, double pages_fetched, BlockNumber pages, + double maxentries, double random_page_cost, double cpu_operator_cost, + double *cost_per_page); + +/* + * doltgres_heap_only_refused answers whether the fork stores a relation, in + * each heap-only contrib module's access method check, which then refuses it + * with its own error. + */ +extern bool doltgres_heap_only_refused(Relation rel); + +/* + * doltgres_visibility_kept answers whether the fork stores a relation, which + * keeps no visibility map and has no pages, in collect_visibility_data, which + * then reads none of its pages, and in pg_visibility's helpers below. + */ +extern bool doltgres_visibility_kept(Relation rel); + +/* + * doltgres_visibility_bits gives a block's visibility map bits in + * pg_visibility's functions: a fork table's are all-visible and all-frozen + * for a block within it and neither past it, and any other relation's are + * the map's own. + */ +extern int doltgres_visibility_bits(Relation rel, BlockNumber blkno, int mapbits); + +/* doltgres_visibility_page is pg_visibility.c's helper that gives a fork table's page-level flag of a block. */ +extern bool doltgres_visibility_page(Relation rel, BlockNumber blkno, Datum *all_visible); + +/* vbits is pg_visibility.c's per-block visibility data, which its helper below fills. */ +struct vbits; + +/* doltgres_visibility_collected is pg_visibility.c's helper that fills a fork table's per-block visibility data. */ +extern void doltgres_visibility_collected(Relation rel, bool include_pd, struct vbits *info); + +/* + * doltgres_tablespace_emptied empties the fork's packs out of a tablespace + * whose Postgres directories hold no files and removes its directory of + * them, in DropTableSpace, waiting, and in tblspc_redo, once before each + * removal of the tablespace's directories. + */ +extern void doltgres_tablespace_emptied(Oid tablespaceoid, bool redo); + /* * doltgres_fold_launch is the fold launcher's background worker: it starts a * fold worker for each database with a storage due a fold. diff --git a/src/include/doltgres/host/delta/directory.h b/src/include/doltgres/host/delta/directory.h index d3f1f4546d89b..c78a35277c2e7 100644 --- a/src/include/doltgres/host/delta/directory.h +++ b/src/include/doltgres/host/delta/directory.h @@ -141,6 +141,16 @@ typedef struct HostDeltaDirectoryDirectoryOps void (*clear) (void *self, uint32 kind, uint32 index); void (*made) (void *self, uint32 kind, uint32 index); void (*unmade) (void *self, uint32 kind, uint32 index); + uint64 (*retired_head) (void *self); + bool (*retired_swap) (void *self, uint64 loaded, uint64 word); + uint32 (*link) (void *self, uint32 slot); + void (*set_link) (void *self, uint32 slot, uint32 value); + uint64 (*record_retired_head) (void *self, uint32 kind); + bool (*record_retired_swap) (void *self, uint32 kind, uint64 loaded, uint64 word); + uint64 (*record_link) (void *self, uint32 kind, uint32 index); + void (*set_record_link) (void *self, uint32 kind, uint32 index, uint64 value); + void (*lock_pass) (void *self); + void (*unlock_pass) (void *self); } HostDeltaDirectoryDirectoryOps; /* HOST_DELTA_DIRECTORY_HINTED is the count of slots each word of the free hint has a bit for. */ @@ -210,6 +220,8 @@ extern void host_delta_directory_woken(HostDeltaDirectoryDirectory directory); #ifdef DOLTGRES_TEST_HOST extern void test_host_delta_directory_tests_a_size_fits_its_word(void); extern void test_host_delta_directory_tests_a_words_index_is_below_its_pages_count(void); +extern void test_host_delta_directory_tests_a_free_pass_loads_only_the_retirements(void); +extern void test_host_delta_directory_tests_a_refused_allocations_pass_loads_only_the_retirements(void); #endif #endif /* DOLTGRES_HOST_DELTA_DIRECTORY_H */ diff --git a/src/include/doltgres/host/lib.h b/src/include/doltgres/host/lib.h index 6579324d5c201..2b5004ecc69b6 100644 --- a/src/include/doltgres/host/lib.h +++ b/src/include/doltgres/host/lib.h @@ -299,6 +299,7 @@ typedef enum HostWalKind HOST_WAL_KIND_TIDS_FREED, HOST_WAL_KIND_MAP_CARRIED, HOST_WAL_KIND_DATABASE_CHANGE, + HOST_WAL_KIND_CARRY_UNLISTED, } HostWalKind; /* @@ -604,6 +605,8 @@ typedef struct HostHostOps bool (*delay_checkpoints) (void *self, const HostMayRaise *raise, const char *const *pauses, size_t pauses_len, HostError *error); + bool (*load_pauses) (void *self, const HostMayRaise *raise, const char *const *pauses, + size_t pauses_len, HostError *error); void (*resume_checkpoints) (void *self, bool finished); bool (*xid_status) (void *self, const HostMayRaise *raise, HostXid xid, HostStatus *result, HostError *error); @@ -772,11 +775,12 @@ typedef struct HostHostOps bool (*backups_running) (void *self); bool (*collection_begin) (void *self); void (*collection_end) (void *self); - bool (*collection_ask) (void *self, const HostMayRaise *raise, uint32 database, uint64 *result, - HostError *error); - bool (*collection_asked) (void *self, uint64 *result_0, uint32 *result_1); + bool (*collection_ask) (void *self, const HostMayRaise *raise, uint32 database, uint32 emptied, + uint64 *result, HostError *error); + bool (*collection_asked) (void *self, uint64 *result_0, uint32 *result_1, uint32 *result_2); void (*collection_answer) (void *self, uint64 word, const uint64 *counts); void (*collector_started) (void *self); + bool (*collector_running) (void *self); bool (*database_locked) (void *self, const HostMayRaise *raise, uint32 database, bool *result, HostError *error); void (*database_unlocked) (void *self, uint32 database); @@ -876,6 +880,7 @@ extern bool host_Storage_PartialEq_eq(const HostStorage *self, const HostStorage extern bool host_Storage_Debug_fmt(const void *value, StdFormatter *f); extern bool host_Xid_PartialEq_eq(const HostXid *self, const HostXid *other); extern bool host_Xid_Debug_fmt(const void *value, StdFormatter *f); +extern bool host_Command_Debug_fmt(const void *value, StdFormatter *f); extern bool host_Row_PartialEq_eq(const HostRow *self, const HostRow *other); extern bool host_Row_Debug_fmt(const void *value, StdFormatter *f); extern bool host_Refused_Debug_fmt(const void *value, StdFormatter *f); @@ -947,6 +952,7 @@ extern bool host_tree_bytes(HostIoIoDoors io, const char *path, size_t path_len, extern bool host_nap(HostIoIoDoors io, StdDuration duration, HostError *error); extern bool host_wal_insert(HostIoIoDoors io, HostWalKind kind, const uint8 *body, size_t body_len, HostLsn *result, HostError *error); +extern bool host_NewCid_Debug_fmt(const void *value, StdFormatter *f); extern bool host_wal_new_cid(HostIoIoDoors io, const HostStorage *storage, HostRow tid, HostNewCid cid, HostLsn *result, HostError *error); extern bool host_wal_flush(HostIoIoDoors io, HostLsn lsn, HostError *error); @@ -1001,6 +1007,7 @@ extern bool host_warn(HostIoIoDoors io, HostError *error_, const char *context, HostError *error); extern bool host_log(HostIoIoDoors io, const char *message, HostError *error); extern void host_refuse_recovery(HostIoIoDoors io, const char *detail, size_t detail_len); +extern bool host_load_pauses(HostIoIoDoors io, const char *const *pauses, size_t pauses_len, HostError *error); extern bool host_delay_checkpoint(HostIoIoDoors io, const char *const *pauses, size_t pauses_len, HostDelayCheckpoint *result, HostError *error); @@ -1022,10 +1029,11 @@ extern void host_backup_end(void); extern bool host_backups_running(void); extern bool host_collection_begin(void); extern void host_collection_end(void); -extern bool host_collection_ask(HostIoIo *io, uint32 database, uint64 *result, HostError *error); -extern bool host_collection_asked(uint64 *result_0, uint32 *result_1); +extern bool host_collection_ask(HostIoIo *io, uint32 database, uint32 emptied, uint64 *result, HostError *error); +extern bool host_collection_asked(uint64 *result_0, uint32 *result_1, uint32 *result_2); extern void host_collection_answer(uint64 word, const uint64 *counts); extern void host_collector_started(void); +extern bool host_collector_running(void); extern bool host_database_locked(HostIoIo *io, uint32 database, bool *result, HostError *error); extern void host_database_unlocked(uint32 database); extern bool host_database_lock_waited(HostIoIo *io, uint32 database, bool *result, HostError *error); diff --git a/src/include/doltgres/host/shared.h b/src/include/doltgres/host/shared.h index b2cbcb66b7918..4ed8a2907d233 100644 --- a/src/include/doltgres/host/shared.h +++ b/src/include/doltgres/host/shared.h @@ -160,9 +160,13 @@ typedef enum HostSharedOpTag HOST_SHARED_OP_LOAD, HOST_SHARED_OP_ADD, HOST_SHARED_OP_EXCHANGE, + HOST_SHARED_OP_LOAD_ACQUIRE, } HostSharedOpTag; -/* HostSharedOp is an operation on an atomic: a load, an add, or an exchange from a value to another. */ +/* + * HostSharedOp is an operation on an atomic: a load, an add, an exchange from + * a value to another, or a load after every read before it. + */ typedef struct HostSharedOp { HostSharedOpTag tag; diff --git a/src/include/doltgres/pg/lib.h b/src/include/doltgres/pg/lib.h index 6c094f6c8dff8..bd002e8c44b71 100644 --- a/src/include/doltgres/pg/lib.h +++ b/src/include/doltgres/pg/lib.h @@ -767,6 +767,7 @@ extern PgIndexFetch *pg_IndexScan_heap_fetch(PgIndexScan *self); extern const PgRel *pg_IndexScan_index(const PgIndexScan *self); extern const PgSnapshot *pg_IndexScan_snapshot(const PgIndexScan *self); extern PgIndexScan *pg_IndexScanDesc_scan(PgIndexScanDesc *self); +extern bool pg_transaction_snapshot_view(const PgMayRaise *raise, PgSnapshotView *result, PgPgError *error); extern bool pg_snapshot_view(const PgMayRaise *raise, const PgSnapshot *snapshot, PgSnapshotView *result, PgPgError *error); extern DgScanKeys pg_ScanKeys_with_count(const PgScanKeys *keys, int32 count); diff --git a/src/include/doltgres/repo/lib.h b/src/include/doltgres/repo/lib.h index 23e325b0a2b89..19d511ba46655 100644 --- a/src/include/doltgres/repo/lib.h +++ b/src/include/doltgres/repo/lib.h @@ -32,11 +32,20 @@ #include "doltgres/std/vec.h" #include "doltgres/trees/store.h" -/* RepoSpilled is Spilled: a transaction's spill file among a checkpoint's runs, by its top-level xid and its number. */ +/* + * RepoSpilled is Spilled: a transaction's spill file among a checkpoint's + * runs, by its top-level xid and its number, for a free space map's spill its + * transaction's abort kept, the point a fold passes it at, and, for a fold's + * residual of a map's spills, the fold's point, from which readers read it. + */ typedef struct RepoSpilled { uint64 xid; uint32 number; + bool has_passed; + uint64 passed; + bool has_from; + uint64 from; } RepoSpilled; /* diff --git a/src/include/doltgres/shmem/collection.h b/src/include/doltgres/shmem/collection.h index 237b4be16ede0..3cfb72d9f2d4d 100644 --- a/src/include/doltgres/shmem/collection.h +++ b/src/include/doltgres/shmem/collection.h @@ -37,8 +37,10 @@ extern bool shmem_collection_backups_running(void); extern bool shmem_collection_collection_begin(void); extern void shmem_collection_collection_end(void); extern void shmem_collection_worker_started(void); -extern bool shmem_collection_ask(const PgMayRaise *raise, uint32 database, uint64 result[3], PgPgError *error); -extern bool shmem_collection_asked_for(std_TupleU64U32_t *result); +extern bool shmem_collection_worker_running(void); +extern bool shmem_collection_ask(const PgMayRaise *raise, uint32 database, uint32 emptied, uint64 result[3], + PgPgError *error); +extern bool shmem_collection_asked_for(std_TupleU64U32_t *result, uint32 *emptied); extern void shmem_collection_answer(uint64 word, const uint64 counts[3]); #endif /* FRONTEND */ diff --git a/src/include/doltgres/shmem/delta.h b/src/include/doltgres/shmem/delta.h index 2b7b1c55586cb..cfa7c41071e0e 100644 --- a/src/include/doltgres/shmem/delta.h +++ b/src/include/doltgres/shmem/delta.h @@ -72,6 +72,12 @@ extern uint64 shmem_delta_overshoot(void); extern bool shmem_delta_overshoot_swap(uint64 loaded, uint64 word); extern void shmem_delta_count_waiter(bool add); extern uint32 shmem_delta_hand(void); +extern uint64 shmem_delta_retired(void); +extern bool shmem_delta_retired_swap(uint64 loaded, uint64 word); +extern uint32 shmem_delta_link(uint32 slot); +extern void shmem_delta_set_link(uint32 slot, uint32 value); +extern void shmem_delta_lock_pass(void); +extern void shmem_delta_unlock_pass(void); extern bool shmem_delta_lock(const PgMayRaise *raise, PgPgError *error); extern void shmem_delta_unlock(void); extern bool shmem_delta_wait(const PgMayRaise *raise, uint64 seen, PgPgError *error); diff --git a/src/include/doltgres/shmem/records.h b/src/include/doltgres/shmem/records.h index c6f71dae6677f..79fe664d7720f 100644 --- a/src/include/doltgres/shmem/records.h +++ b/src/include/doltgres/shmem/records.h @@ -102,9 +102,13 @@ typedef enum ShmemRecordsOpTag SHMEM_RECORDS_OP_LOAD, SHMEM_RECORDS_OP_ADD, SHMEM_RECORDS_OP_EXCHANGE, + SHMEM_RECORDS_OP_LOAD_ACQUIRE, } ShmemRecordsOpTag; -/* ShmemRecordsOp is an operation on an atomic: a load, an add, or an exchange of the second value for the first. */ +/* + * ShmemRecordsOp is an operation on an atomic: a load, an add, an exchange of + * the second value for the first, or a load after a read barrier. + */ typedef struct ShmemRecordsOp { ShmemRecordsOpTag tag; @@ -132,6 +136,8 @@ extern bool shmem_records_pool(size_t kind, std_TupleU32U32_t *result); extern uint64 shmem_records_word(uint32 at); extern bool shmem_records_swap(uint32 at, uint64 loaded, uint64 word); extern uint32 shmem_records_hand(size_t kind); +extern uint64 shmem_records_retired(size_t kind); +extern bool shmem_records_retired_swap(size_t kind, uint64 loaded, uint64 word); extern void shmem_records_clear(size_t kind, uint32 index); extern bool shmem_records_lock(const PgMayRaise *raise, size_t kind, uint32 index, size_t lock, bool exclusive, PgPgError *error); diff --git a/src/include/doltgres/simc/simc.h b/src/include/doltgres/simc/simc.h index fcd7c7ced2a17..b146094c1eb82 100644 --- a/src/include/doltgres/simc/simc.h +++ b/src/include/doltgres/simc/simc.h @@ -41,6 +41,12 @@ typedef struct SimcTestIo void *process; } SimcTestIo; +/* SimcProcess is sim's Process: a hold of the test process, a Box the drop releases. */ +typedef struct SimcProcess +{ + void *f0; +} SimcProcess; + /* SimcTaken is sim's Taken: a snapshot a test took, which simc_Taken_lent lends, a Box. */ typedef struct SimcTaken { @@ -150,6 +156,26 @@ typedef struct simc_TupleSqlstateString_t StdString f1; } simc_TupleSqlstateString_t; +/* simc_TupleU8VecU8Lsn_t is the tuple (u8, Vec, Lsn): a logged record's kind's value, its body, and its end. */ +typedef struct simc_TupleU8VecU8Lsn_t +{ + uint8 f0; + StdVec f1; + HostLsn f2; +} simc_TupleU8VecU8Lsn_t; + +/* + * simc_TupleStorageRowNewCidLsn_t is the tuple (Storage, Row, NewCid, Lsn): a + * logged new cid record's storage, its row, its command ids, and its end. + */ +typedef struct simc_TupleStorageRowNewCidLsn_t +{ + HostStorage f0; + HostRow f1; + HostNewCid f2; + HostLsn f3; +} simc_TupleStorageRowNewCidLsn_t; + /* * SimcReport is sim's Report: the run's outcome, each exit work that failed, * and the Rust's report, a Box, which holds the rest and the test @@ -165,6 +191,8 @@ typedef struct SimcReport extern SimcTestIo simc_io(void); extern void simc_TestIo_drop(void *self); +extern SimcProcess simc_process(void); +extern void simc_Process_drop(void *self); extern bool simc_tempdir(SimcTempDir *result, StdIoError *error); extern const char *simc_TempDir_path(const SimcTempDir *self, size_t *len); extern void simc_TempDir_drop(void *self); @@ -176,15 +204,22 @@ extern void simc_set_disk_space(uint64 free, uint64 total); extern void simc_refuse_owned_append(uint64 after); extern void simc_refuse_open(HostDeltaPagePage name, uint64 after); extern void simc_refuse_record(uint64 shape, uint64 location, uint64 after); +extern void simc_refuse_any_open(uint64 after); extern void simc_refuse_any_take(uint64 after); extern void simc_refuse_nothing(void); extern HostXid simc_begin(void); extern HostXid simc_begin_under(HostXid parent); extern void simc_commit(HostXid xid); extern void simc_abort(HostXid xid); +extern void simc_truncate_commit_log(HostXid oldest); +extern void simc_savepoint(void); +extern void simc_rollback_to(void); +extern void simc_release(void); +extern void simc_next_command(void); extern SimcTaken simc_historic(HostXid xmin, HostXid xmax, const HostXid *committed, size_t committed_len, const HostXid *own, size_t own_len, HostCommand command); extern SimcTaken simc_snapshot(HostSnapshotKind kind, bool recovery); +extern SimcTaken simc_non_vacuumable(HostXid horizon); extern HostSnapshot simc_Taken_lent(const SimcTaken *self); extern void simc_Taken_drop(void *self); extern size_t simc_scratch_files(void); @@ -197,6 +232,8 @@ extern StdPathBuf simc_data_dir(void); extern StdVec simc_take_warnings(void); extern StdVec simc_take_logs(void); extern bool simc_kept_wal_read(HostLsn *from, HostLsn *to); +extern StdVec simc_logged(void); +extern StdVec simc_new_cids(void); extern SimcSim simc_Sim_start(const char *root, size_t root_len, uint64 seed); extern void simc_Sim_stop(const SimcSim *self); extern StdVec simc_Sim_trace(const SimcSim *self); @@ -214,11 +251,13 @@ extern SimcReport simc_Run_finish(SimcRun self); extern void simc_Run_drop(void *self); extern void simc_Report_drop(void *self); extern bool simc_Report_Debug_fmt(const void *value, StdFormatter *f); +extern StdVec simc_Report_trace(const SimcReport *report); extern bool simc_Outcome_Debug_fmt(const void *value, StdFormatter *f); extern bool simc_Ending_Debug_fmt(const void *value, StdFormatter *f); extern bool simc_Ending_PartialEq_eq(const SimcEnding *self, const SimcEnding *other); extern void simc_size_delta(uint64 budget); extern uint64 simc_delta_holds(void); +extern size_t simc_free_slots(void); extern uint64 simc_delta_waiting(void); extern uint64 simc_delta_wakes(void); extern void simc_unfreed(uint64 shape, size_t *records, size_t *pages); diff --git a/src/include/doltgres/store/collect.h b/src/include/doltgres/store/collect.h index 3d021511ce625..5dbee4c1b2cb1 100644 --- a/src/include/doltgres/store/collect.h +++ b/src/include/doltgres/store/collect.h @@ -116,7 +116,7 @@ typedef struct store_collect_held_t } store_collect_held_t; extern bool store_collect_collected(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, - uint32 database, uint64 limit, uint64 percent, const StdVec *old, + uint32 database, uint64 limit, uint64 percent, uint32 emptied, const StdVec *old, bool *has_result, StoreCollectCollected *result, HostError *error); extern void store_collect_Collected_drop(StoreCollectCollected *self); extern void store_collect_Schedule_Default_default(StoreCollectSchedule *result); @@ -126,7 +126,8 @@ extern bool store_collect_collect_due(HostIoIo *io, StoreCollectSchedule *schedu const StoreCollectSettings *settings, HostError *error); extern bool store_collect_collect_asked(HostIoIo *io, StoreCollectSchedule *schedule, const StoreCollectDatabase *databases, size_t databases_len, uint32 id, - const StoreCollectSettings *settings, uint64 *result, HostError *error); + uint32 emptied, const StoreCollectSettings *settings, uint64 *result, + HostError *error); #ifdef DOLTGRES_TEST_STORE extern void test_store_collect_tests_a_collection_retires_a_pack_nothing_names_and_keeps_a_named_one(void); diff --git a/src/include/doltgres/trees/cursor.h b/src/include/doltgres/trees/cursor.h index 80f6457bb806f..0349e3bb6a382 100644 --- a/src/include/doltgres/trees/cursor.h +++ b/src/include/doltgres/trees/cursor.h @@ -162,6 +162,8 @@ extern size_t trees_cursor_Cursor_index(const TreesCursorCursor *self); extern StdBytes trees_cursor_Cursor_key(const TreesCursorCursor *self); extern StdBytes trees_cursor_Cursor_value(const TreesCursorCursor *self); extern TreesStoreTree trees_cursor_Cursor_child_tree(const TreesCursorCursor *self); +extern bool trees_cursor_Cursor_find_in_node(TreesCursorCursor *self, HostIoIo *io, const uint8 *key, size_t key_len, + const TreesKeyOrder *order, bool *result, ChunksErrorError *error); extern bool trees_cursor_Cursor_stamps(const TreesCursorCursor *self, HostIoIo *io, TreesPinPin *pin, StdVec *result, ChunksErrorError *error); extern bool trees_cursor_Cursor_values(const TreesCursorCursor *self, HostIoIo *io, TreesPinPin *pin, StdVec *result, @@ -189,6 +191,9 @@ extern void trees_cursor_Cursor_drop(TreesCursorCursor *self); extern TreesCursorLeafStamps trees_cursor_LeafStamps_Default_default(void); extern bool trees_cursor_LeafStamps_of(TreesCursorLeafStamps *self, HostIoIo *io, TreesPinPin *pin, const TreesCursorCursor *cursor, StdVec *result, ChunksErrorError *error); +extern bool trees_cursor_LeafStamps_stamp_of(TreesCursorLeafStamps *self, HostIoIo *io, TreesPinPin *pin, + const TreesCursorCursor *cursor, uint64 tuple, uint64 *copies, bool *some, + ChunksTrailerStamp *result, StdVec *values, ChunksErrorError *error); extern void trees_cursor_LeafStamps_drop(TreesCursorLeafStamps *self); #endif /* DOLTGRES_TREES_CURSOR_H */ diff --git a/src/include/doltgres/txn/base.h b/src/include/doltgres/txn/base.h index 1054d8b050f5f..c56ba7737eee8 100644 --- a/src/include/doltgres/txn/base.h +++ b/src/include/doltgres/txn/base.h @@ -102,7 +102,7 @@ typedef struct TxnBaseKey * largest top-level xid it folded, the epoch its rows are stored at, its * tuple id tree, the tuple id counter and the fresh end, the free space map, * its sizes, its key, its table's and index's object ids, the newest xid that - * ended a row it applied, and its table's pending transforms. + * ended a row it applied, its table's pending transforms, and its read point. */ typedef struct TxnBaseBase { @@ -151,6 +151,13 @@ typedef struct TxnBaseBase */ bool has_chain; TreesStoreTree chain; + + /* + * read is the read point, the point readers decide by which delta and + * spilled versions the base applies, below a moved base's fold point and + * equal to it on every other base. + */ + uint64 read; } TxnBaseBase; /* @@ -214,6 +221,13 @@ extern void txn_base_pending_value(uint64 ender, uint32 heap_bytes, const uint32 extern bool txn_base_pending_read(const uint8 *value, size_t value_len, uint64 *result_0, uint32 *result_1, StdVec *result_2, HostError *error); +/* + * TXN_BASE_OWN is OWN, the bit of a pending tuple id's ender word that marks + * a row its own writer ended, whose free waits for the horizon to pass that + * writer and logs no conflict horizon; full xids stay below it. + */ +#define TXN_BASE_OWN (UINT64CONST(1) << 63) + #ifdef DOLTGRES_TEST_TXN extern void txn_base_Base_Clone_clone(const TxnBaseBase *self, TxnBaseBase *result); extern TxnBaseBase txn_base_tests_full(void); diff --git a/src/include/doltgres/txn/blobs.h b/src/include/doltgres/txn/blobs.h index c551e886152d4..f413f8862fd8d 100644 --- a/src/include/doltgres/txn/blobs.h +++ b/src/include/doltgres/txn/blobs.h @@ -81,4 +81,12 @@ extern bool txn_blobs_written(HostIoIo *io, const HostSharedHeld *deltas, uint32 extern bool txn_blobs_redo(HostIoIo *io, const HostSharedHeld *deltas, TxnRecordChunkWrite chunk, uint64 lsn, HostError *error); +#ifdef DOLTGRES_TEST_TXN +extern void test_txn_blobs_tests_a_logged_write_logs_a_chunk_an_unlogged_one_left_and_redo_makes_it_durable(void); +extern void test_txn_blobs_tests_a_blob_in_the_delta_reads_back_through_blobs(void); +extern void test_txn_blobs_tests_a_transactions_chunks_spill_read_back_and_leave_at_its_abort(void); +extern void test_txn_blobs_tests_a_chunk_another_running_transaction_holds_stays_after_that_ones_abort(void); +extern void test_txn_blobs_tests_a_rolled_back_chunk_takes_nothing_from_its_tops_count(void); +#endif + #endif /* DOLTGRES_TXN_BLOBS_H */ diff --git a/src/include/doltgres/txn/change.h b/src/include/doltgres/txn/change.h index 15b0d587f58b9..c1af7d102aef1 100644 --- a/src/include/doltgres/txn/change.h +++ b/src/include/doltgres/txn/change.h @@ -121,4 +121,11 @@ extern bool txn_change_end(HostIoIo *io, const HostSharedHeld *held, TxnChangeEn extern bool txn_change_redo(HostIoIo *io, const HostSharedHeld *held, const TxnRecordRowChange *change, uint64 lsn, HostError *error); +#ifdef DOLTGRES_TEST_TXN +extern void test_txn_change_tests_a_keys_stripe_is_its_storage_and_bytes_or_its_storage_alone(void); +extern void test_txn_change_tests_a_delete_is_logged_pushed_and_redone_once_and_a_lock_is_never_logged(void); +extern void test_txn_change_tests_an_aborted_delete_and_lock_go_at_the_take_back(void); +extern void test_txn_change_tests_a_change_of_a_spilled_row_reads_its_spilled_version(void); +#endif + #endif /* DOLTGRES_TXN_CHANGE_H */ diff --git a/src/include/doltgres/txn/entry.h b/src/include/doltgres/txn/entry.h index 4b8b52d92876c..61944f50bb4d6 100644 --- a/src/include/doltgres/txn/entry.h +++ b/src/include/doltgres/txn/entry.h @@ -90,4 +90,14 @@ extern bool txn_entry_unique(HostIoIo *io, TreesStoreNodeStore store, const Host extern bool txn_entry_based(const uint8 *value, size_t value_len, const ChunksTrailerStamp *stamps, size_t stamps_len, bool keyless, StdBytes *result, HostError *error); +#ifdef DOLTGRES_TEST_TXN +extern void test_txn_entry_tests_an_index_entrys_changes_are_logged_pushed_and_redone_once(void); +extern void test_txn_entry_tests_an_index_base_entry_reads_back_and_refuses_a_keyed_stamp_and_an_uneven_count(void); +extern void test_txn_entry_tests_an_index_storages_large_keys_spill_at_the_threshold(void); +extern void test_txn_entry_tests_a_unique_entry_takes_a_free_key_and_answers_an_existing_entrys_value(void); +extern void test_txn_entry_tests_a_counted_unique_entry_writes_nothing_for_its_own_copy_and_answers_another(void); +extern void test_txn_entry_tests_a_partial_unique_check_answers_a_running_write_without_waiting(void); +extern void test_txn_entry_tests_an_entry_and_its_record_change_together(void); +#endif + #endif /* DOLTGRES_TXN_ENTRY_H */ diff --git a/src/include/doltgres/txn/fold.h b/src/include/doltgres/txn/fold.h index b21ba37a6f942..2fa58109b6db0 100644 --- a/src/include/doltgres/txn/fold.h +++ b/src/include/doltgres/txn/fold.h @@ -120,6 +120,7 @@ extern const TxnVisibleRuleOps TXN_FOLD_FOLDING_RULE_OPS; /* TXN_FOLD_COMMITTING_RULE_OPS is Committing's Rule. */ extern const TxnVisibleRuleOps TXN_FOLD_COMMITTING_RULE_OPS; +extern const TxnVisibleRuleOps TXN_FOLD_EVERY_RULE_OPS; /* * TxnFoldCtidEdit is CtidEdit: a fold's edit of its base's tuple id tree, a @@ -282,7 +283,7 @@ extern bool txn_fold_folded_key(HostIoIo *io, TreesStoreNodeStore store, bool ha const TreesKeyOrder *order, bool keyless, bool index, TxnVisibleRule rule, const uint8 *key, size_t key_len, TxnMemtableVersionVersion *versions, size_t versions_len, bool *has_result, TxnFoldKeyFold *result, uint64 *max_ended, - HostError *error); + StdVec *gone, HostError *error); extern void txn_fold_FoldedRows_drop(TxnFoldFoldedRows *self); extern bool txn_fold_folded_rows(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, bool has_base, TreesStoreTree base, bool has_tuple_ids, TreesStoreTree tuple_ids, @@ -290,7 +291,8 @@ extern bool txn_fold_folded_rows(HostIoIo *io, TreesStoreNodeStore store, const TxnFoldFoldedRows *result, HostError *error); extern bool txn_fold_folded(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, const TreesKeyOrder *order, bool keyless, HostLsn point, - TxnFoldReform *reform, TxnFoldReached *reached, bool *result, HostError *error); + const TxnInsertPage *page, TxnFoldReform *reform, TxnFoldReached *reached, bool *result, + HostError *error); extern bool txn_fold_caught_up(HostIoIo *io, const HostSharedHeld *held, const TxnInsertPage *page, uint64 upto, TxnFoldCaught *result, HostError *error); extern bool txn_fold_Applied_PartialEq_eq(const TxnFoldApplied *self, const TxnFoldApplied *other); @@ -307,16 +309,16 @@ extern bool txn_fold_chained(HostIoIo *io, const RepoRepo *repo, const HostShare const HostSharedHeld *held, TxnBasePending *pending, HostError *error); extern bool txn_fold_committed_moves(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, bool has_base, TreesStoreTree base, bool has_tuple_ids, TreesStoreTree tuple_ids, - const TreesKeyOrder *order, bool keyless, bool index, uint64 own, - const uint8 *key, size_t key_len, TxnBaseSizes *result_0, + const TreesKeyOrder *order, bool keyless, bool index, HostLsn folded, + uint64 own, const uint8 *key, size_t key_len, TxnBaseSizes *result_0, TxnBaseSizes *result_1, HostError *error); extern bool txn_fold_committed_value_moves(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, bool has_root, TreesStoreTree root, uint64 own, const uint8 *key, size_t key_len, TxnBaseSizes *result_0, TxnBaseSizes *result_1, HostError *error); extern bool txn_fold_copied(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, - const HostSharedHeld *held, TxnBaseBase *base, HostLsn point, bool has_nodes, - uint64 nodes, TxnFoldCopies *memo, bool *result, HostError *error); + const HostSharedHeld *held, TxnBaseBase *base, HostLsn point, HostLsn read, + bool has_nodes, uint64 nodes, TxnFoldCopies *memo, bool *result, HostError *error); extern void txn_fold_Copies_drop(TxnFoldCopies *self); extern bool txn_fold_fold_now(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, const TreesKeyOrder *order, bool keyless, bool has_point, @@ -332,7 +334,7 @@ extern bool txn_fold_redo(HostIoIo *io, const RepoRepo *repo, const HostSharedRe HostError *error); extern bool txn_fold_redo_chained(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, const TxnBasePending *entries, size_t entries_len, - TxnBaseBase *recorded, HostError *error); + TxnBaseBase *recorded, HostLsn end, HostError *error); extern bool txn_fold_settled(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostSharedHeld *held, StdVec *result, HostError *error); extern bool txn_fold_startup_fold(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, diff --git a/src/include/doltgres/txn/identity.h b/src/include/doltgres/txn/identity.h index 118a9ba5d33c4..88cfcd15e7193 100644 --- a/src/include/doltgres/txn/identity.h +++ b/src/include/doltgres/txn/identity.h @@ -122,8 +122,21 @@ extern bool txn_identity_removed(HostIoIo *io, HostIoWaits *waits, uint32 reposi TxnMemtableVersionWriter writer, TxnStatusRoomed roomed, StdVec *keys, HostError *error); extern bool txn_identity_folded(HostIoIo *io, uint32 repository, HostLsn point, StdVec *result, HostError *error); -extern bool txn_identity_dropped(HostIoIo *io, uint32 repository, HostLsn point, HostError *error); +extern bool txn_identity_dropped(HostIoIo *io, uint32 repository, HostLsn point, bool *took, HostError *error); extern bool txn_identity_redo(HostIoIo *io, TxnRecordDatabaseChange *change, uint32 own, uint64 lsn, HostError *error); +#ifdef DOLTGRES_TEST_TXN +extern void test_txn_identity_tests_an_objects_id_derives_from_its_kind_parent_and_name(void); +extern void test_txn_identity_tests_a_columns_id_holds_its_types_category(void); +extern void test_txn_identity_tests_a_pairs_keys_find_it_from_either_side(void); +extern void test_txn_identity_tests_a_tables_pairs_read_back_by_its_oid_and_its_id(void); +extern void test_txn_identity_tests_an_aborted_transactions_pairs_go_at_the_take_back(void); +extern void test_txn_identity_tests_a_taken_id_moves_on_and_a_removed_pair_frees_it(void); +extern void test_txn_identity_tests_redone_identity_changes_give_the_same_pairs_once(void); +extern void test_txn_identity_tests_an_identity_change_reads_back_as_written(void); +extern void test_txn_identity_tests_a_saves_fold_keeps_its_committed_pairs_and_leaves_running_ones(void); +extern void test_txn_identity_tests_a_saves_drop_takes_its_folded_pairs_out(void); +#endif + #endif /* DOLTGRES_TXN_IDENTITY_H */ diff --git a/src/include/doltgres/txn/insert.h b/src/include/doltgres/txn/insert.h index 55dca549abe5e..52138f016e49a 100644 --- a/src/include/doltgres/txn/insert.h +++ b/src/include/doltgres/txn/insert.h @@ -194,6 +194,21 @@ typedef enum TxnInsertRechecked } TxnInsertRechecked; /* TxnInsertBase is Base: what allocation reads of the base, its free space tree, fresh end, and generation. */ +/* TxnInsertCarriedPut is a carry's put of another writer: its tuple id, its heap bytes, and its writer. */ +typedef struct TxnInsertCarriedPut +{ + HostRow tid; + uint64 heap_bytes; + TxnMemtableVersionWriter writer; +} TxnInsertCarriedPut; + +/* TxnInsertCarriedTake is a carried put's take: its block and the take's version's record. */ +typedef struct TxnInsertCarriedTake +{ + uint32 block; + StdVec record; +} TxnInsertCarriedTake; + typedef struct TxnInsertBase { bool has_tree; @@ -205,8 +220,8 @@ typedef struct TxnInsertBase /* * TxnInsertPlaced is Placed: a known tuple id's place in its block's free * space map entry, read outside any window, for place_taken to take inside a - * window: the tuple id, the row's room, the base read, the counter read, and - * the base's entry of the block. + * window: the tuple id, the row's room, the base read, the counter read, the + * block's entry, and the base's entry of the block. */ typedef struct TxnInsertPlaced { @@ -214,6 +229,13 @@ typedef struct TxnInsertPlaced TxnInsertRoom room; TxnInsertBase base; uint64 counter; + bool has_current; + + /* + * current is a Vec, the block's newest entry, which place_taken's + * take compares (bugs row 342). + */ + StdVec current; bool has_below; /* below is a Vec, the base's entry of the block. */ StdVec below; @@ -296,9 +318,14 @@ extern bool txn_insert_Placed_read(HostIoIo *io, TreesStoreNodeStore store, cons HostError *error); extern TxnMemtableMapTake txn_insert_Placed_take(const TxnInsertPlaced *self, HostSharedRecord record); extern void txn_insert_Placed_drop(TxnInsertPlaced *self); +extern void txn_insert_carried_takes_drop(StdVec *takes); +extern bool txn_insert_carried_takes(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, + const TxnInsertCarriedPut *puts, size_t puts_len, const TxnInsertPage *page, + StdVec *result, HostError *error); extern bool txn_insert_place_taken(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t take, - HostSharedRecord record, TxnMemtableVersionWriter writer, - const TxnInsertPlaced *placed, HostError *error); + HostSharedRecord record, TxnMemtableVersionWriter writer, bool has_roomed, + TxnStatusRoomed roomed, const TxnInsertPlaced *placed, bool *moved, + HostError *error); extern bool txn_insert_Room_fits(const TxnInsertRoom *self, uint64 free, uint16 offset); extern uint64 txn_insert_Room_need(const TxnInsertRoom *self); extern bool txn_insert_insert(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, @@ -326,4 +353,41 @@ extern bool txn_insert_redo_spent(HostIoIo *io, TreesStoreNodeStore store, const extern bool txn_insert_redo_left(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, const TxnRecordPageLeft *left, const TxnInsertPage *page, HostError *error); +#ifdef DOLTGRES_TEST_TXN +extern bool txn_insert_tests_taken(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, + TxnInsertInsert *row, const TxnInsertRoom *room, const HostSharedHeld *toast, + const TxnValuesEntry *toast_entries, size_t toast_entries_len, uint64 *result, + HostError *error); +extern HostStorage txn_insert_tests_stored(uint32 database); +extern TxnInsertInsert txn_insert_tests_row(HostStorage storage, uint64 n); +extern void test_txn_insert_tests_inserts_take_heaps_tuple_ids_by_offsets_and_by_bytes(void); +extern void test_txn_insert_tests_a_row_the_last_page_cant_hold_goes_to_the_lowest_page_left_with_room(void); +extern void test_txn_insert_tests_redo_leaves_the_free_space_map_and_the_counter_as_the_primarys(void); +extern void test_txn_insert_tests_a_take_redone_again_changes_nothing(void); +extern void test_txn_insert_tests_an_insert_maps_its_out_of_line_values_in_its_window(void); +extern void test_txn_insert_tests_a_gathering_from_an_inserts_prepare_to_its_push_keeps_the_counter(void); +extern void test_txn_insert_tests_an_insert_that_leaves_the_fresh_page_and_fails_redoes_as_the_primary(void); +extern void test_txn_insert_tests_a_row_whose_value_references_sit_in_an_overflow_chunk_redoes_its_map_entries(void); +extern void test_txn_insert_tests_an_insert_that_fails_after_its_chunks_maps_no_value(void); +extern void test_txn_insert_tests_redo_from_any_point_gives_a_row_with_all_its_map_entries_or_none(void); +extern void test_txn_insert_tests_a_kept_values_count_add_is_redone_once(void); +extern void test_txn_insert_tests_a_load_moves_the_counter_past_every_loaded_tid_and_left_page(void); +extern void test_txn_insert_tests_an_insert_logical_decoding_reads_logs_heaps_record_of_its_command(void); +extern void test_txn_insert_tests_a_checkpoints_runs_and_every_record_after_redo_as_the_primary(void); +extern void test_txn_insert_tests_a_window_between_a_sweeps_rows_and_its_cut_leaves_its_take_without_its_row(void); +extern void test_txn_insert_tests_windows_after_a_sweeps_cut_are_in_its_runs_with_neither_their_rows_nor_their_takes(void); +extern void test_txn_insert_tests_an_aborted_inserts_value_map_versions_go_at_its_take_back_and_its_free_space_take_stays(void); +extern void test_txn_insert_tests_an_update_keeps_its_key_or_moves_it_and_redoes_as_the_primary(void); +extern void test_txn_insert_tests_a_keyless_or_deferrable_update_names_its_rows_by_their_tuple_ids(void); +extern void test_txn_insert_tests_an_updates_combo_command_shows_on_the_row_it_ends(void); +extern void test_txn_insert_tests_a_speculative_insert_holds_its_token_until_confirmed(void); +extern void test_txn_insert_tests_an_aborted_updates_new_row_and_end_go_at_the_take_back(void); +extern void test_txn_insert_tests_a_resolve_whose_stripe_moves_three_times_holds_its_writers_back(void); +extern void test_txn_insert_tests_a_checkpoint_between_an_updates_count_adds_and_its_row_leaves_the_counts_once(void); +extern void test_txn_insert_tests_a_checkpoint_before_or_after_an_update_leaves_its_kept_counts_once(void); +extern void test_txn_insert_tests_a_crash_after_a_key_change_whose_runs_hold_one_chain_redoes_both(void); +extern void test_txn_insert_tests_a_resolve_whose_first_target_is_refused_ends_in_bounded_turns(void); +extern void test_txn_insert_tests_a_frozen_inserts_row_change_replays_with_no_status_room(void); +#endif + #endif /* DOLTGRES_TXN_INSERT_H */ diff --git a/src/include/doltgres/txn/memtable/branch.h b/src/include/doltgres/txn/memtable/branch.h index b85f33b974172..d791fea42233a 100644 --- a/src/include/doltgres/txn/memtable/branch.h +++ b/src/include/doltgres/txn/memtable/branch.h @@ -71,11 +71,25 @@ typedef TxnMemtableTablePreparedState TxnMemtableTablePreparedBranchDeltas; #define TXN_MEMTABLE_BRANCH_DELTAS_LOCK_INDEX 0 #define TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_WRITING_INDEX 1 +/* The delta's long lock IdentitySpillsWriting, the writers' lock of its identity spills' manifest (bugs row 291). */ +#define TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_WRITING_INDEX 2 + /* The delta's atomics of its chunk spills' manifest: ChunkSpills, ChunkSpillsPending, and ChunkSpillsChanged. */ #define TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_INDEX 3 #define TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_PENDING_INDEX 4 #define TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_CHANGED_INDEX 5 +/* TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_TAKEN_INDEX is ChunkSpillsTaken's atomic: the take generation of the delta's chunk spills. */ +#define TXN_MEMTABLE_BRANCH_CHUNK_SPILLS_TAKEN_INDEX 7 + +/* The delta's atomics of its identity spills' manifest: IdentitySpills, IdentitySpillsPending, and IdentitySpillsChanged. */ +#define TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_INDEX 8 +#define TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_PENDING_INDEX 9 +#define TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_CHANGED_INDEX 10 + +/* TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_TAKEN_INDEX is IdentitySpillsTaken's atomic: the take generation of the delta's identity spills. */ +#define TXN_MEMTABLE_BRANCH_IDENTITY_SPILLS_TAKEN_INDEX 11 + /* TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS is Identity's Tree: the memtable of a working state's identity tree. */ extern const TxnMemtableTableTreeOps TXN_MEMTABLE_BRANCH_IDENTITY_TREE_OPS; @@ -88,6 +102,9 @@ extern const TxnMemtableTableTreeOps TXN_MEMTABLE_BRANCH_CHUNKS_TREE_OPS; /* TXN_MEMTABLE_BRANCH_CHUNK_WORD_INDEX is ChunkWord's atomic: the word naming a working state's chunks' memtable. */ #define TXN_MEMTABLE_BRANCH_CHUNK_WORD_INDEX 1 +/* TXN_MEMTABLE_BRANCH_IDENTITY_WORD is IdentityWord's atomic: the word naming a working state's identity memtable. */ +#define TXN_MEMTABLE_BRANCH_IDENTITY_WORD 2 + extern void txn_memtable_branch_node_key(const uint8 *address, uint32 tablespace, uint32 relation, uint64 point, uint8 *result); extern bool txn_memtable_branch_pieces(const uint8 *node, uint64 writer, const uint8 *value, size_t value_len, diff --git a/src/include/doltgres/txn/memtable/branch_template.h b/src/include/doltgres/txn/memtable/branch_template.h index 14ccadf6f6ae0..4143259abe32a 100644 --- a/src/include/doltgres/txn/memtable/branch_template.h +++ b/src/include/doltgres/txn/memtable/branch_template.h @@ -175,7 +175,7 @@ TXN_MEMTABLE_BRANCH_NAME(txn_memtable_branch_identity_prepare_body) (void *frame frame->ok = txn_memtable_table_prepared(frame->io, frame->aims.ptr, frame->aims.len, false, none, TXN_MEMTABLE_BRANCH_NAME(txn_memtable_branch_inside), frame->captures, TXN_MEMTABLE_BRANCH_NAME(txn_memtable_branch_dropped), &has_unstored, - &unstored, frame->error); + &unstored, NULL, frame->error); TXN_MEMTABLE_BRANCH_NAME(txn_memtable_branch_identity_prepare_pad) (frame); } diff --git a/src/include/doltgres/txn/memtable/index.h b/src/include/doltgres/txn/memtable/index.h index b7dffac47e60a..976830754903c 100644 --- a/src/include/doltgres/txn/memtable/index.h +++ b/src/include/doltgres/txn/memtable/index.h @@ -42,6 +42,12 @@ /* TXN_MEMTABLE_INDEX_NODE is NODE, the most words a node takes: its head, its place word, and a link at each level. */ #define TXN_MEMTABLE_INDEX_NODE (2 + TXN_MEMTABLE_INDEX_HEIGHT) +/* TXN_MEMTABLE_INDEX_WORDS is WORDS, the count of a node's words before its links: its head and its place word. */ +#define TXN_MEMTABLE_INDEX_WORDS UINT64CONST(2) + +/* TXN_MEMTABLE_INDEX_LEAST_WORDS is the words of txn_memtable_index_least's page, which holds a full-height node alone. */ +#define TXN_MEMTABLE_INDEX_LEAST_WORDS (1 + TXN_MEMTABLE_INDEX_WORDS + TXN_MEMTABLE_INDEX_HEIGHT) + /* * TxnMemtableIndexClosesOps is the Closes trait: a shape whose records close, * the lock of its first list, which opens a record for changes, and its diff --git a/src/include/doltgres/txn/memtable/map.h b/src/include/doltgres/txn/memtable/map.h index 3b78fe3da87b6..ffa0fe28053e2 100644 --- a/src/include/doltgres/txn/memtable/map.h +++ b/src/include/doltgres/txn/memtable/map.h @@ -27,6 +27,7 @@ #include "doltgres/host/shared.h" #include "doltgres/std/vec.h" #include "doltgres/txn/memtable/version.h" +#include "doltgres/txn/status.h" /* TXN_MEMTABLE_MAP_KEY is KEY, the length of a map's key, a u32 in big-endian order. */ #define TXN_MEMTABLE_MAP_KEY 4 @@ -66,14 +67,34 @@ typedef enum TxnMemtableMapStep TXN_MEMTABLE_MAP_STEP_REFUSE, } TxnMemtableMapStep; -/* TxnMemtableMapTaken is Taken's variants: what a take did; a took's T comes beside it. */ +/* + * TxnMemtableMapTaken is Taken's variants: what a take did, moved when the + * entry its finish found or the base generation wasn't its caller's read; a + * took's T comes beside it. + */ typedef enum TxnMemtableMapTaken { TXN_MEMTABLE_MAP_TAKEN_TOOK, TXN_MEMTABLE_MAP_TAKEN_RETRY, TXN_MEMTABLE_MAP_TAKEN_REFUSED, + TXN_MEMTABLE_MAP_TAKEN_MOVED, } TxnMemtableMapTaken; +/* + * TxnMemtableMapRead is what a take's caller read of its key outside the + * window, which its step computes from: the key's entry, or none, and the + * state's base generation read with it. A take with a read answers moved + * when the entry its finish finds or the generation differs, as a spill or a + * fold between them makes it (bugs row 342). + */ +typedef struct TxnMemtableMapRead +{ + bool has_entry; + const uint8 *entry; + size_t entry_len; + uint64 generation; +} TxnMemtableMapRead; + /* TxnMemtableMapEntry is Entry, an Option>: a key's newest entry in a map, its value, or none for a removal. */ typedef struct TxnMemtableMapEntry { @@ -130,19 +151,31 @@ extern const struct TxnMemtableTableTreeOps TXN_MEMTABLE_MAP_VALUES_TREE_OPS; extern bool txn_memtable_map_Map_Debug_fmt(const void *value, StdFormatter *f); extern bool txn_memtable_map_named(HostIoTakesShort io, struct TxnMemtableTablePreparedState *prepared, size_t take, - TxnMemtableVersionWriter writer, bool has_lsn, uint64 lsn, HostRow tuple, - StdVec *values, HostError *error); + TxnMemtableVersionWriter writer, bool has_roomed, TxnStatusRoomed roomed, + bool has_lsn, uint64 lsn, HostRow tuple, StdVec *values, HostError *error); extern size_t txn_memtable_map_named_most(size_t count); +extern bool txn_memtable_map_kept(HostIoTakesShort io, struct TxnMemtableTablePreparedState *prepared, size_t take, + HostSharedRecord state, bool has_base, const uint8 *base, size_t base_len, + const TxnMemtableMapRead *read, bool *result, HostError *error); extern bool txn_memtable_map_valued(const TxnMemtableVersionVersion *version, bool *some, StdBytes *result, HostError *error); extern bool txn_memtable_map_versions(const struct TxnMemtableTableTreeOps *tree, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, uint32 key, StdVec *result, HostError *error); +extern bool txn_memtable_map_in_memtable(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 key, + bool *some, TxnMemtableMapEntry *result, HostError *error); +extern bool txn_memtable_map_spilled_newest(HostIoIo *io, HostSharedRecord state, uint32 key, bool *some, + TxnMemtableMapEntry *result, HostError *error); extern bool txn_memtable_map_newest(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 key, bool *some, TxnMemtableMapEntry *result, HostError *error); +extern bool txn_memtable_map_taker(const TxnMemtableVersionVersion *version, + txn_TupleWriterRowEntryU64TupleVecU32Bool_t *taker, HostError *error); +extern bool txn_memtable_map_spilled_takes(HostIoIo *io, HostSharedRecord state, uint32 block, StdVec *result, + HostError *error); extern bool txn_memtable_map_takers(HostIoIo *io, HostSharedRecord state, TxnMemtableMapMap map, uint32 key, StdVec *result, HostError *error); -extern bool txn_memtable_map_ended(HostIoIo *io, HostSharedRecord state, HostRow tid, uint32 ender, HostError *error); +extern bool txn_memtable_map_ended(HostIoIo *io, HostSharedRecord state, HostRow tid, uint64 writer, uint32 ender, + HostError *error); extern void txn_memtable_map_takers_drop(StdVec *takers); extern HostRow txn_memtable_map_taken(TxnMemtableVersionOpRef op); extern bool txn_memtable_map_marked(HostIoIo *io, HostSharedRecord state, uint32 block, uint64 lsn, StdVec *result, @@ -156,9 +189,10 @@ extern bool txn_memtable_map_raised(HostSharedRecord state, const uint8 *key, si extern bool txn_memtable_map_cut(HostIoIo *io, HostSharedRecord state, bool *some, txn_TupleU64VecVecU8_t *result, HostError *error); extern bool txn_memtable_map_took_core(HostIoTakesShort io, struct TxnMemtableTablePreparedState *prepared, - size_t take, TxnMemtableVersionWriter writer, bool has_lsn, uint64 lsn, + size_t take, TxnMemtableVersionWriter writer, bool has_roomed, + TxnStatusRoomed roomed, bool has_lsn, uint64 lsn, bool has_claim, HostRow claim_tid, uint64 claim_bytes, bool has_base, - const uint8 *base, size_t base_len, + const uint8 *base, size_t base_len, const TxnMemtableMapRead *read, TxnMemtableMapStep (*step) (void *captures, bool has_current, const uint8 *current, size_t current_len, StdVec *out), @@ -176,6 +210,7 @@ extern void test_txn_memtable_map_tests_a_seek_reads_on_after_a_writer_follows_i extern void test_txn_memtable_map_tests_one_window_takes_in_the_rows_storage_and_its_toast_relations(void); extern void test_txn_memtable_map_tests_a_cut_reads_the_counter_with_its_blocks_entry_from_the_last_range(void); extern void test_txn_memtable_map_tests_a_map_only_prepare_takes_its_entry_and_makes_no_rows_memtable(void); +extern void test_txn_memtable_map_tests_a_take_whose_read_moved_answers_moved_and_pushes_nothing(void); extern void test_txn_memtable_map_tests_the_free_space_maps_takes_newest_and_seek(void); extern void test_txn_memtable_map_tests_the_value_maps_takes_newest_and_seek(void); extern void test_txn_memtable_map_tests_a_maps_seek_goes_on_past_a_ranges_end(void); diff --git a/src/include/doltgres/txn/memtable/map_template.h b/src/include/doltgres/txn/memtable/map_template.h index f2b4d03263877..a5e1648811b3f 100644 --- a/src/include/doltgres/txn/memtable/map_template.h +++ b/src/include/doltgres/txn/memtable/map_template.h @@ -81,9 +81,12 @@ extern bool TXN_MEMTABLE_MAP_CAT(txn_memtable_map_finish_, TXN_MEMTABLE_MAP_TS) TxnMemtableTablePreparedState *prepared, size_t take, TxnMemtableVersionWriter writer, + bool has_roomed, + TxnStatusRoomed roomed, bool has_lsn, uint64 lsn, bool has_base, const uint8 *base, size_t base_len, + const TxnMemtableMapRead *read, TxnMemtableMapStep (*step) (void *captures, bool has_current, const uint8 *current, @@ -103,10 +106,12 @@ extern bool TXN_MEMTABLE_MAP_CAT(txn_memtable_map_took_, TXN_MEMTABLE_MAP_TS) (H TxnMemtableTablePreparedState *prepared, size_t take, TxnMemtableVersionWriter writer, + bool has_roomed, TxnStatusRoomed roomed, bool has_lsn, uint64 lsn, bool has_claim, HostRow claim_tid, uint64 claim_bytes, bool has_base, const uint8 *base, size_t base_len, + const TxnMemtableMapRead *read, TxnMemtableMapStep (*step) (void *captures, bool has_current, const uint8 *current, @@ -186,7 +191,8 @@ TXN_MEMTABLE_MAP_CAT(txn_memtable_map_prepare_, TXN_MEMTABLE_MAP_RS) (HostIoIo * return txn_memtable_table_readied(io, false, none, NULL, 0, takes, takes_len, false, TXN_MEMTABLE_MAP_RNAME(txn_memtable_map_inside), &captures, - TXN_MEMTABLE_MAP_RNAME(txn_memtable_map_dropped), TXN_MEMTABLE_MAP_RESULT_ERROR); + TXN_MEMTABLE_MAP_RNAME(txn_memtable_map_dropped), NULL, + TXN_MEMTABLE_MAP_RESULT_ERROR); } #undef TXN_MEMTABLE_MAP_RNAME @@ -284,6 +290,8 @@ typedef struct TXN_MEMTABLE_MAP_TNAME (txn_memtable_map_took_frame_t) TxnMemtableTablePreparedState *prepared; size_t take; TxnMemtableVersionWriter writer; + bool has_roomed; + TxnStatusRoomed roomed; bool has_lsn; uint64 lsn; bool has_claim; @@ -292,6 +300,7 @@ typedef struct TXN_MEMTABLE_MAP_TNAME (txn_memtable_map_took_frame_t) bool has_base; const uint8 *base; size_t base_len; + const TxnMemtableMapRead *read; TXN_MEMTABLE_MAP_TNAME(txn_memtable_map_took_t) captures; TxnMemtableMapTaken *result; @@ -319,9 +328,10 @@ TXN_MEMTABLE_MAP_TNAME(txn_memtable_map_took_body) (void *frame_) { TXN_MEMTABLE_MAP_TNAME(txn_memtable_map_took_frame_t) * frame = frame_; - frame->ok = txn_memtable_map_took_core(frame->io, frame->prepared, frame->take, frame->writer, frame->has_lsn, - frame->lsn, frame->has_claim, frame->claim_tid, frame->claim_bytes, - frame->has_base, frame->base, frame->base_len, + frame->ok = txn_memtable_map_took_core(frame->io, frame->prepared, frame->take, frame->writer, frame->has_roomed, + frame->roomed, frame->has_lsn, frame->lsn, frame->has_claim, + frame->claim_tid, frame->claim_bytes, + frame->has_base, frame->base, frame->base_len, frame->read, TXN_MEMTABLE_MAP_TNAME(txn_memtable_map_took_step), TXN_MEMTABLE_MAP_TNAME(txn_memtable_map_took_tuple), TXN_MEMTABLE_MAP_TNAME(txn_memtable_map_took_then), &frame->captures, @@ -335,8 +345,10 @@ bool TXN_MEMTABLE_MAP_CAT(txn_memtable_map_finish_, TXN_MEMTABLE_MAP_TS) (HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t take, TxnMemtableVersionWriter writer, + bool has_roomed, TxnStatusRoomed roomed, bool has_lsn, uint64 lsn, bool has_base, const uint8 *base, size_t base_len, + const TxnMemtableMapRead *read, TxnMemtableMapStep (*step) (void *captures, bool has_current, const uint8 *current, @@ -354,8 +366,9 @@ TXN_MEMTABLE_MAP_CAT(txn_memtable_map_finish_, TXN_MEMTABLE_MAP_TS) (HostIoTakes { TXN_MEMTABLE_MAP_TNAME(txn_memtable_map_took_frame_t) frame = { - .io = io, .prepared = prepared, .take = take, .writer = writer, .has_lsn = has_lsn, .lsn = lsn, - .has_base = has_base, .base = base, .base_len = base_len, + .io = io, .prepared = prepared, .take = take, .writer = writer, .has_roomed = has_roomed, .roomed = roomed, + .has_lsn = has_lsn, .lsn = lsn, + .has_base = has_base, .base = base, .base_len = base_len, .read = read, .captures = { .step = step, .step_captures = step_captures, .step_drop = step_drop, .then = then, @@ -374,10 +387,12 @@ TXN_MEMTABLE_MAP_CAT(txn_memtable_map_finish_, TXN_MEMTABLE_MAP_TS) (HostIoTakes bool TXN_MEMTABLE_MAP_CAT(txn_memtable_map_took_, TXN_MEMTABLE_MAP_TS) (HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t take, - TxnMemtableVersionWriter writer, bool has_lsn, + TxnMemtableVersionWriter writer, bool has_roomed, + TxnStatusRoomed roomed, bool has_lsn, uint64 lsn, bool has_claim, HostRow claim_tid, uint64 claim_bytes, bool has_base, const uint8 *base, size_t base_len, + const TxnMemtableMapRead *read, TxnMemtableMapStep (*step) (void *captures, bool has_current, const uint8 *current, @@ -399,9 +414,10 @@ TXN_MEMTABLE_MAP_CAT(txn_memtable_map_took_, TXN_MEMTABLE_MAP_TS) (HostIoTakesSh { TXN_MEMTABLE_MAP_TNAME(txn_memtable_map_took_frame_t) frame = { - .io = io, .prepared = prepared, .take = take, .writer = writer, .has_lsn = has_lsn, .lsn = lsn, + .io = io, .prepared = prepared, .take = take, .writer = writer, .has_roomed = has_roomed, .roomed = roomed, + .has_lsn = has_lsn, .lsn = lsn, .has_claim = has_claim, .claim_tid = claim_tid, .claim_bytes = claim_bytes, .has_base = has_base, - .base = base, .base_len = base_len, + .base = base, .base_len = base_len, .read = read, .captures = { .step = step, .step_captures = step_captures, .step_drop = step_drop, .tuple = tuple, diff --git a/src/include/doltgres/txn/memtable/pages.h b/src/include/doltgres/txn/memtable/pages.h index 42860388c0548..b0eb0694680d9 100644 --- a/src/include/doltgres/txn/memtable/pages.h +++ b/src/include/doltgres/txn/memtable/pages.h @@ -42,6 +42,9 @@ /* TXN_MEMTABLE_PAGES_FIRST is FIRST, the first word a range page hands out, past its four counts. */ #define TXN_MEMTABLE_PAGES_FIRST UINT64CONST(4) +/* TXN_MEMTABLE_PAGES_SMALLEST is SMALLEST, the body of a range's first page, before what it must hold makes it larger. */ +#define TXN_MEMTABLE_PAGES_SMALLEST (UINT64CONST(1) << 10) + /* TXN_MEMTABLE_PAGES_DROPPED is DROPPED, what a range's list stores over the index of a page the remover dropped. */ #define TXN_MEMTABLE_PAGES_DROPPED PG_UINT64_MAX diff --git a/src/include/doltgres/txn/memtable/table.h b/src/include/doltgres/txn/memtable/table.h index 27f5bbfe853a6..06c7461650eb6 100644 --- a/src/include/doltgres/txn/memtable/table.h +++ b/src/include/doltgres/txn/memtable/table.h @@ -101,6 +101,14 @@ typedef struct TxnMemtableTableReserve bool has_seen; TxnMemtableSplitSeen seen; uint64 grown; + + /* + * counting is whether a push by the step within runs counts at its + * writer's entry. + */ + bool counting; + /* counted is whether the step pushed a version it counted. */ + bool counted; } TxnMemtableTableReserve; /* TxnMemtableTableTarget is Target: one push a prepare readied, its tree, where its hand-off found its key, and its reserve. */ @@ -144,10 +152,27 @@ typedef struct TxnMemtableTablePreparedState HostDeltaHold hold; const TxnMemtableTableAim *aims; size_t aims_len; + + /* + * first is the place among the caller's aims of aims' first: a prepare + * readies the caller's aims in groups an empty budget can give, aims and + * targets being the group's (bugs row 338). + */ + size_t first; /* targets is a Vec. */ StdVec targets; bool has_tuple; TxnMemtableTableReservation tuple; + + /* + * kept is the aim whose key's open range its caller keeps locked, + * kept_open that range under kept_guard, on which every finish of the + * aim's key runs while has_kept (bugs row 342). + */ + bool has_kept; + size_t kept; + HostSharedGuard kept_guard; + TxnMemtableIndexKeyedOpen kept_open; } TxnMemtableTablePreparedState; /* TxnMemtableTableCompactable is Compactable: a page a range names alone that the remover found worth compacting, its index in the range's list, its name, and its copy's size. */ @@ -166,12 +191,13 @@ typedef struct TxnMemtableTableRemoving size_t handles_len; } TxnMemtableTableRemoving; -/* txn_TupleHandleU64_t is a version's handle with its writer's top-level xid, (Handle, u64), as copied hands them out. */ -typedef struct txn_TupleHandleU64_t +/* txn_TupleHandleU64Bool_t is a version's handle with its writer's top-level xid and whether it is killed, (Handle, u64, bool), as copied hands them out. */ +typedef struct txn_TupleHandleU64Bool_t { TxnMemtableVersionHandle f0; uint64 f1; -} txn_TupleHandleU64_t; + bool f2; +} txn_TupleHandleU64Bool_t; /* TXN_MEMTABLE_TABLE_STATE_OWNER_OPS is State's Owner: a storage's state. */ extern const TxnMemtableTableOwnerOps TXN_MEMTABLE_TABLE_STATE_OWNER_OPS; @@ -219,8 +245,6 @@ extern bool txn_memtable_table_finish_row(HostIoTakesShort io, TxnMemtableTableP const TxnMemtableVersionChange *change, bool has_roomed, TxnStatusRoomed roomed, TxnMemtableVersionHandle *result, HostError *error); -extern bool txn_memtable_table_counted(HostIoTakesShort io, const TxnMemtableTablePreparedState *prepared, - bool has_roomed, TxnStatusRoomed roomed, HostError *error); extern bool txn_memtable_table_newest(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t row, bool *some, TxnMemtableVersionHandle *result, HostError *error); extern bool txn_memtable_table_holds(HostIoIo *io, HostSharedRecord state, const uint8 *key, size_t key_len, @@ -238,7 +262,6 @@ extern bool txn_memtable_table_removed(const TxnMemtableTableTreeOps *tree, Host extern bool txn_memtable_table_pruned(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, const uint8 *key, size_t key_len, bool has_aborting, uint64 aborting, HostError *error); -extern bool txn_memtable_table_uncounted(HostIoIo *io, const TxnMemtableVersionVersion *version, HostError *error); extern bool txn_memtable_table_waited(HostIoIo *io, HostDeltaFull full, bool spilling, HostError *error); extern bool txn_memtable_table_versions(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, const uint8 *key, size_t key_len, StdVec *result, HostError *error); @@ -271,6 +294,11 @@ extern bool txn_memtable_table_Reserve_push(TxnMemtableTableReserve *self, TxnMe const HostDeltaHold *hold, const uint8 *key, size_t key_len, const TxnMemtableVersionChange *change, TxnMemtableVersionHandle *result, HostError *error); +extern bool txn_memtable_table_Reserve_push_found(TxnMemtableTableReserve *self, TxnMemtableIndexKeyedOpen *open, + const HostDeltaHold *hold, TxnMemtableIndexFound found, + const uint8 *key, size_t key_len, + const TxnMemtableVersionChange *change, + TxnMemtableVersionHandle *result, HostError *error); extern size_t txn_memtable_table_Prepared_target(const TxnMemtableTablePreparedState *self, size_t take); extern bool txn_memtable_table_Prepared_aimed(const TxnMemtableTablePreparedState *self, size_t target, HostSharedRecord *result_0, const TxnMemtableTableTreeOps **result_1, @@ -278,14 +306,39 @@ extern bool txn_memtable_table_Prepared_aimed(const TxnMemtableTablePreparedStat extern bool txn_memtable_table_Prepared_tuple(TxnMemtableTablePreparedState *self, const HostDeltaHold **result_0, HostSharedRecord *result_1, TxnMemtableTableReservation *result_2, HostError *error); + +/* + * TXN_MEMTABLE_TABLE_WHOLE is the most aims a prepare readies as one group at + * every delta_memory, so a caller of a few aims never meets a group (bugs row + * 338). + */ +#define TXN_MEMTABLE_TABLE_WHOLE 8 + +/* + * TxnMemtableTableRefusal is a carry's prepare's answer of the delta + * budget's refusal: whether the budget refused it, having taken nothing, and + * the refusal, which the carry waits on. + */ +typedef struct TxnMemtableTableRefusal +{ + bool refused; + HostDeltaFull full; +} TxnMemtableTableRefusal; + extern bool txn_memtable_table_prepared(HostIoIo *io, const TxnMemtableTableAim *aims, size_t aims_len, bool has_tuple, TxnMemtableTableReservation tuple, bool (*inside) (void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, HostError *error), void *inside_captures, void (*inside_drop) (void *captures), bool *has_unstored, TxnMemtableTableReservation *unstored, - HostError *error); + TxnMemtableTableRefusal *refused, HostError *error); +extern bool txn_memtable_table_refused(HostIoIo *io, HostDeltaFull full, TxnMemtableTableRefusal *refused, + bool *result, HostError *error); +extern bool txn_memtable_table_kept(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t index, + HostError *error); +extern void txn_memtable_table_unkept(TxnMemtableTablePreparedState *prepared); extern bool txn_memtable_table_within(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t index, + bool has_roomed, TxnStatusRoomed roomed, bool (*step) (void *captures, TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, const uint8 *key, size_t key_len, TxnMemtableTableReserve *reserve, HostError *error), @@ -293,8 +346,12 @@ extern bool txn_memtable_table_within(HostIoTakesShort io, TxnMemtableTablePrepa extern bool txn_memtable_table_settled(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, TxnMemtableTableAt at, bool has_seen, const TxnMemtableSplitSeen *seen, HostError *error); +#ifdef DOLTGRES_TEST_TXN +extern void test_txn_memtable_table_tests_a_refused_try_leaves_no_memtable_range_or_page_behind(void); +extern void test_txn_memtable_table_tests_a_refused_try_of_two_trees_leaves_neither_memtable_behind(void); extern bool txn_memtable_table_first(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, HostError *error); +#endif extern bool txn_memtable_table_memtable(const TxnMemtableTableTreeOps *tree, HostSharedRecord owner, uint64 *result); extern bool txn_memtable_table_names(const TxnMemtableTableTreeOps *tree, HostSharedRecord owner, uint64 location); extern bool txn_memtable_table_abandoned(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, @@ -307,11 +364,11 @@ extern bool txn_memtable_table_readied(HostIoIo *io, bool has_rows, HostSharedRe bool tuple, bool (*inside) (void *captures, HostIoIo *io, TxnMemtableTablePreparedState *prepared, HostError *error), void *inside_captures, - void (*inside_drop) (void *captures), + void (*inside_drop) (void *captures), TxnMemtableTableRefusal *refused, HostError *error); extern bool txn_memtable_table_pushed(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t index, - const TxnMemtableVersionChange *change, TxnMemtableVersionHandle *result, - HostError *error); + const TxnMemtableVersionChange *change, bool has_roomed, + TxnStatusRoomed roomed, TxnMemtableVersionHandle *result, HostError *error); extern bool txn_memtable_table_finished(HostIoTakesShort io, TxnMemtableTablePreparedState *prepared, size_t index, const TxnMemtableVersionChange *change, bool has_roomed, TxnStatusRoomed roomed, TxnMemtableVersionHandle *result, @@ -334,7 +391,7 @@ extern bool txn_memtable_table_held(const TxnMemtableTableTreeOps *tree, HostIoI const uint8 *key, size_t key_len, uint64 lsn, bool *result, HostError *error); extern bool txn_memtable_table_alone(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, const uint8 *key, size_t key_len, const TxnMemtableVersionChange *change, - TxnMemtableVersionHandle *result, + bool has_roomed, TxnStatusRoomed roomed, TxnMemtableVersionHandle *result, HostError *error); extern bool txn_memtable_table_removed_all(const TxnMemtableTableTreeOps *tree, HostIoIo *io, HostSharedRecord owner, const TxnMemtableTableRemoving *keys, size_t keys_len, StdVec *result, @@ -346,7 +403,7 @@ extern bool txn_memtable_table_copied(const TxnMemtableTableTreeOps *tree, HostI bool killed, bool (*each) (void *captures, HostIoIo *io, const uint8 *key, size_t key_len, const StdVec *records, size_t records_len, - const txn_TupleHandleU64_t *handles, + const txn_TupleHandleU64Bool_t *handles, size_t handles_len, HostError *error), void *captures, HostError *error); diff --git a/src/include/doltgres/txn/memtable/table_template.h b/src/include/doltgres/txn/memtable/table_template.h index 4628fc077f540..ed7512fd51108 100644 --- a/src/include/doltgres/txn/memtable/table_template.h +++ b/src/include/doltgres/txn/memtable/table_template.h @@ -140,7 +140,7 @@ TXN_MEMTABLE_TABLE_NAME(txn_memtable_table_prepare) (HostIoIo *io, HostSharedRec return txn_memtable_table_readied(io, true, state, rows, rows_len, takes, takes_len, tuple, TXN_MEMTABLE_TABLE_NAME(txn_memtable_table_inside), &captures, - TXN_MEMTABLE_TABLE_NAME(txn_memtable_table_dropped), + TXN_MEMTABLE_TABLE_NAME(txn_memtable_table_dropped), NULL, TXN_MEMTABLE_TABLE_RESULT_ERROR); } @@ -172,7 +172,7 @@ TXN_MEMTABLE_TABLE_NAME(txn_memtable_table_prepare_striped) (HostIoIo *io, HostI return txn_memtable_table_readied(io, true, state, rows, rows_len, takes, takes_len, tuple, TXN_MEMTABLE_TABLE_NAME(txn_memtable_table_inside), &captures, - TXN_MEMTABLE_TABLE_NAME(txn_memtable_table_dropped), + TXN_MEMTABLE_TABLE_NAME(txn_memtable_table_dropped), NULL, TXN_MEMTABLE_TABLE_RESULT_ERROR); } #endif diff --git a/src/include/doltgres/txn/memtable/tuples.h b/src/include/doltgres/txn/memtable/tuples.h index 2dc24e7691013..036075ef17ac8 100644 --- a/src/include/doltgres/txn/memtable/tuples.h +++ b/src/include/doltgres/txn/memtable/tuples.h @@ -38,7 +38,8 @@ extern bool txn_memtable_tuples_get(HostIoIo *io, HostSharedRecord state, HostRo extern bool txn_memtable_tuples_emptied(HostIoIo *io, HostSharedRecord state, HostError *error); extern bool txn_memtable_tuples_reserve(HostIoIo *io, HostSharedRecord state, const uint8 *key, size_t key_len, - TxnMemtableTableReservation *result, HostError *error); + TxnMemtableTableReservation *result, TxnMemtableTableRefusal *refused, + HostError *error); extern bool txn_memtable_tuples_given(HostIoIo *io, HostSharedRecord state, HostError *error); extern bool txn_memtable_tuples_retired(HostIoIo *io, HostSharedRecord state, HostError *error); extern bool txn_memtable_tuples_dropped(HostIoIo *io, HostSharedRecord state, const uint64 *tids, size_t tids_len, @@ -52,6 +53,10 @@ extern void test_txn_memtable_tuples_tests_a_reservation_no_finish_stored_is_giv extern void test_txn_memtable_tuples_tests_two_tids_of_one_first_slot_are_each_found(void); extern void test_txn_memtable_tuples_tests_a_tables_first_pages_are_the_smallest_that_hold_them(void); extern void test_txn_memtable_tuples_tests_a_tables_tids_are_found_after_inserts_and_a_growth(void); +extern void test_txn_memtable_tuples_tests_a_table_of_mostly_dropped_tids_is_compacted_at_its_load_factor(void); +extern void test_txn_memtable_tuples_tests_a_compaction_with_a_reservation_outstanding_moves_no_byte_page(void); +extern void test_txn_memtable_tuples_tests_a_writer_waits_on_a_compactions_exclusive_lock(void); +extern void test_txn_memtable_tuples_tests_a_compaction_failed_at_any_step_leaves_every_tid_found(void); /* txn_memtable_tuples_tests_stored_t is stored's prepare's captures: the tid and the change. */ typedef struct txn_memtable_tuples_tests_stored_t diff --git a/src/include/doltgres/txn/memtable/version.h b/src/include/doltgres/txn/memtable/version.h index 467ab40296d7e..ad9fb42675111 100644 --- a/src/include/doltgres/txn/memtable/version.h +++ b/src/include/doltgres/txn/memtable/version.h @@ -330,6 +330,13 @@ extern bool txn_memtable_version_Pushed_Debug_fmt(const void *value, StdFormatte extern bool txn_memtable_version_Handle_named(uint64 value, TxnMemtableVersionHandle *result); extern TxnMemtableVersionHandle txn_memtable_version_Handle_spilled(uint64 xid, uint32 number, uint32 place); #ifdef DOLTGRES_TEST_TXN +extern void test_txn_memtable_version_tests_every_ops_record_reads_back_as_written(void); +extern void test_txn_memtable_version_tests_a_stored_record_is_refused_unless_encode_writes_it(void); +extern void test_txn_memtable_version_tests_a_link_holds_its_offset_and_older_handle_at_their_limits(void); +extern void test_txn_memtable_version_tests_a_keys_versions_read_newest_first_with_their_deleters(void); +extern void test_txn_memtable_version_tests_a_version_on_a_fresh_page_is_named_by_its_index_in_the_list(void); +extern void test_txn_memtable_version_tests_a_hint_and_a_token_clear_racing_both_survive(void); +extern void test_txn_memtable_version_tests_a_shown_combo_is_the_rows_raw_command_and_no_token(void); extern uint64 txn_memtable_version_Handle_page(TxnMemtableVersionHandle self); #endif extern size_t txn_memtable_version_encoded(const TxnMemtableVersionWriter *writer, const TxnMemtableOp *op, @@ -367,6 +374,7 @@ extern uint32 txn_memtable_version_Version_raw(const TxnMemtableVersionVersion * extern bool txn_memtable_version_Version_ender(const TxnMemtableVersionVersion *self, uint32 *result); extern void txn_memtable_version_Version_end(TxnMemtableVersionVersion *self, uint32 ender); extern void txn_memtable_version_Version_set_hint(TxnMemtableVersionVersion *self, TxnMemtableVersionHint hint); +extern bool txn_memtable_version_Version_counted(const TxnMemtableVersionVersion *self); extern bool txn_memtable_version_Version_killed(const TxnMemtableVersionVersion *self); extern TxnMemtableOp txn_memtable_version_OpRef_owned(const TxnMemtableVersionOpRef *self); extern void txn_memtable_version_Version_stored(const TxnMemtableVersionVersion *self, StdVec *result); @@ -387,12 +395,19 @@ extern bool txn_memtable_version_chain(const TxnMemtableIndexKeyedOps *keyed, extern bool txn_memtable_version_Chain_Iterator_next(TxnMemtableVersionChain *self, bool *has_result, TxnMemtableVersionVersion *result, HostError *error); extern bool txn_memtable_version_push(TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, const uint8 *key, - size_t key_len, const TxnMemtableVersionChange *change, bool *has_fresh, + size_t key_len, const TxnMemtableVersionChange *change, bool counted, + bool *has_fresh, HostDeltaOwned *fresh, StdVec *spare, StdVec *buffer, TxnMemtableVersionPushed *result, HostError *error); +extern bool txn_memtable_version_push_found(TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, + TxnMemtableIndexFound found, const uint8 *key, size_t key_len, + const TxnMemtableVersionChange *change, bool counted, bool *has_fresh, + HostDeltaOwned *fresh, StdVec *spare, StdVec *buffer, + TxnMemtableVersionPushed *result, HostError *error); extern bool txn_memtable_version_removed(const TxnMemtableIndexKeyedOpen *open, const HostDeltaHold *hold, TxnMemtableIndexNode node, const TxnMemtableVersionHandle *handles, - size_t handles_len, bool *taken, size_t taken_len, bool *result_0, + size_t handles_len, bool *taken, size_t taken_len, StdVec *counted, + bool *result_0, StdVec *result_1, uint64 *result_2, HostError *error); extern TxnMemtableVersionAddresses txn_memtable_version_PutRef_addresses(const TxnMemtableVersionPutRef *self); @@ -406,5 +421,6 @@ extern void txn_memtable_version_Version_confirm(TxnMemtableVersionVersion *self extern void txn_memtable_version_Version_kill(TxnMemtableVersionVersion *self); extern void txn_memtable_version_Version_drop(TxnMemtableVersionVersion *self); extern void txn_memtable_version_versions_drop(StdVec *versions, size_t from); +extern void txn_memtable_version_ended(StdVec *record, uint32 ender); #endif /* DOLTGRES_TXN_MEMTABLE_VERSION_H */ diff --git a/src/include/doltgres/txn/outcome.h b/src/include/doltgres/txn/outcome.h index 968154c398413..a980246e3a36b 100644 --- a/src/include/doltgres/txn/outcome.h +++ b/src/include/doltgres/txn/outcome.h @@ -29,4 +29,8 @@ extern bool txn_outcome_outcome(HostIoIo *io, TxnMemtableVersionVersion *version HostError *error); extern bool txn_outcome_stamped(HostIoIo *io, uint64 xid, HostStatus *result, HostError *error); +#ifdef DOLTGRES_TEST_TXN +extern void test_txn_outcome_tests_a_writers_outcome_answers_frozen_with_no_door_and_caches_an_ended_writers(void); +#endif + #endif /* DOLTGRES_TXN_OUTCOME_H */ diff --git a/src/include/doltgres/txn/record.h b/src/include/doltgres/txn/record.h index a7b0f6b7115b7..2ceb371210d55 100644 --- a/src/include/doltgres/txn/record.h +++ b/src/include/doltgres/txn/record.h @@ -242,6 +242,18 @@ typedef struct TxnRecordMapCarried StdVec versions; } TxnRecordMapCarried; +/* + * TxnRecordCarryUnlisted is CarryUnlisted: a carry whose spill files its + * primary took back out of its new storage's manifest after a count's add + * failed, named by the ends of the carry's records, ascending. + */ +typedef struct TxnRecordCarryUnlisted +{ + HostStorage storage; + /* ends is a Vec. */ + StdVec ends; +} TxnRecordCarryUnlisted; + /* TxnRecordStorageEmptied is StorageEmptied: a storage emptied in place by a truncate. */ typedef struct TxnRecordStorageEmptied { @@ -261,6 +273,13 @@ typedef struct TxnRecordTidsFreed StdVec runs; /* dead is a Vec<(u32, u16, u16, u64)>, as runs. */ StdVec dead; + + /* + * enders is a Vec: each of runs' tuple ids' stored ender, then each + * of dead's, in tuple id order, which a replayed free's take-out matches + * (bugs row 320). + */ + StdVec enders; /* values is the TOAST storage and each value id's count, or none. */ bool has_values; host_TupleStorageVecTupleU32U64_t values; @@ -355,6 +374,10 @@ extern void txn_record_ChunkWrite_encode(const TxnRecordChunkWrite *self, StdVec extern bool txn_record_ChunkWrite_pieces(TxnRecordChunkWrite *self, StdVec *result, HostError *error); extern void txn_record_BaseAdvanced_encode(const TxnRecordBaseAdvanced *self, StdVec *result); extern void txn_record_StorageEmptied_encode(const TxnRecordStorageEmptied *self, StdVec *result); +extern void txn_record_CarryUnlisted_encode(const TxnRecordCarryUnlisted *self, StdVec *result); +extern bool txn_record_CarryUnlisted_decode(const uint8 *body, size_t body_len, TxnRecordCarryUnlisted *result, + HostError *error); +extern void txn_record_CarryUnlisted_drop(TxnRecordCarryUnlisted *self); extern void txn_record_PageLeft_encode(const TxnRecordPageLeft *self, StdVec *result); extern void txn_record_PageLeft_encode_with(const TxnRecordPageLeft *self, ChunksRecordRecordWriter *w, StdVec *result); @@ -364,4 +387,19 @@ extern void txn_record_DatabaseChange_encode(const TxnRecordDatabaseChange *self extern void txn_record_MapCarried_encode(const TxnRecordMapCarried *self, StdVec *result); extern void txn_record_TidsFreed_encode(const TxnRecordTidsFreed *self, StdVec *result); +#ifdef DOLTGRES_TEST_TXN +extern void test_txn_record_tests_a_row_change_reads_back_and_refuses_every_other_form(void); +extern void test_txn_record_tests_a_page_leave_reads_back_and_refuses_every_other_form(void); +extern void test_txn_record_tests_a_built_bases_node_reads_back_with_its_stamps(void); +extern void test_txn_record_tests_stamps_on_a_blob_or_of_another_count_are_refused(void); +extern void test_txn_record_tests_a_moves_carried_free_space_reads_back_and_refuses_every_other_form(void); +extern void test_txn_record_tests_a_chunk_reads_back_and_refuses_every_other_form(void); +extern void test_txn_record_tests_an_updates_kept_values_read_back_in_their_values_order(void); +extern void test_txn_record_tests_a_folds_record_reads_back_and_refuses_every_other_form(void); +extern void test_txn_record_tests_a_free_reads_back_and_refuses_runs_out_of_bounds_or_order(void); +extern void test_txn_record_tests_a_frees_taken_down_values_read_back_and_refuse_every_other_form(void); +extern void test_txn_record_tests_an_index_entrys_change_reads_back_and_refuses_every_field_its_op_doesnt_carry(void); +extern void test_txn_record_tests_a_built_nodes_key_names_its_base_and_its_fields_come_together(void); +#endif + #endif /* DOLTGRES_TXN_RECORD_H */ diff --git a/src/include/doltgres/txn/rewrite.h b/src/include/doltgres/txn/rewrite.h index 19dd492832985..86005c5a67041 100644 --- a/src/include/doltgres/txn/rewrite.h +++ b/src/include/doltgres/txn/rewrite.h @@ -142,6 +142,18 @@ typedef struct txn_TupleU32TupleEntryU64_t txn_TupleEntryU64_t f1; } txn_TupleU32TupleEntryU64_t; +/* + * TxnRewriteCarry is Carry: a version a rewrite, a move, or an index build + * carries into its new storage, its row change, under its own writer, with + * the command it showed and the row lock it held. + */ +typedef struct TxnRewriteCarry +{ + TxnRecordRowChange f0; + TxnMemtableVersionShown f1; + TxnMemtableLockMode f2; +} TxnRewriteCarry; + extern bool txn_rewrite_based(HostIoIo *io, TreesStoreNodeStore store, bool has_root, TreesStoreTree root, StdVec *result, HostError *error); extern bool txn_rewrite_keys(HostIoIo *io, const HostSharedHeld *held, const TxnRewriteBased *based, @@ -152,8 +164,11 @@ extern bool txn_rewrite_new_base(StdVec *based, StdVec *copied, bool keyless, ui const host_TupleRowU32_t *carried, size_t carried_len, TxnFoldTrees *result_0, TxnBaseBase *result_1, HostError *error); extern bool txn_rewrite_carried(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, - const TxnRecordRowChange *change, TxnMemtableVersionShown shown, - TxnMemtableLockMode lock, bool logged, const TxnInsertPage *page, HostError *error); + const TxnRecordMapCarried *spaces, size_t spaces_len, const TxnRewriteCarry *carries, + size_t carries_len, bool logged, const TxnInsertPage *page, uint16 most, + HostError *error); +extern bool txn_rewrite_moved_point(HostIoIo *io, HostLsn folded, const uint64 *writers, size_t writers_len, + HostLsn *result, HostError *error); /* txn_rewrite_value_map's values are in ascending order of id, each id once, as the BTreeMap's. */ extern bool txn_rewrite_value_map(const txn_TupleU32TupleEntryU64_t *values, size_t values_len, diff --git a/src/include/doltgres/txn/runs.h b/src/include/doltgres/txn/runs.h index 69ec6a6a008fc..98ca0ac188d91 100644 --- a/src/include/doltgres/txn/runs.h +++ b/src/include/doltgres/txn/runs.h @@ -56,11 +56,23 @@ typedef enum TxnRunsKind /* A database's identity tree's versions, framed as a rows run's. */ TXN_RUNS_KIND_IDENTITY = 10, + /* + * A storage's free space map's own versions a transaction spilled, read + * back as a free space run's. + */ + TXN_RUNS_KIND_SPILLED_FREE = 11, + /* * The chunks of a transaction's blobs it spilled from its repository's * working state's delta. */ TXN_RUNS_KIND_SPILLED_CHUNKS = 12, + + /* + * A database's identity tree's own versions a transaction spilled, read + * back as an identity run's (bugs row 291). + */ + TXN_RUNS_KIND_SPILLED_IDENTITY = 13, } TxnRunsKind; /* TXN_RUNS_FILTER_WORDS is FILTER_WORDS, the most words of a page of a spill's filter: the node cache's smallest frame. */ @@ -180,20 +192,31 @@ extern void txn_runs_spill_name(const TxnRunsNames *names, uint32 number, StdStr extern TxnRunsWriting *txn_runs_Writing_spill(const char *dir, size_t dir_len, const TxnRunsNames *names, TxnRunsTail tail, uint32 number); extern StdBytes txn_runs_Writing_path(const TxnRunsWriting *self); -extern bool txn_runs_Writing_key(TxnRunsWriting *self, HostIoIo *io, const uint8 *key, size_t key_len, +extern bool txn_runs_Writing_key(TxnRunsWriting *self, HostIoIoDoors io, const uint8 *key, size_t key_len, const StdBytes *records, size_t records_len, HostError *error); -extern bool txn_runs_Writing_end(TxnRunsWriting *self, HostIoIo *io, const uint64 *filter, size_t filter_len, +extern bool txn_runs_Writing_end(TxnRunsWriting *self, HostIoIoDoors io, const uint64 *filter, size_t filter_len, bool *some, RepoRunFile *result, HostError *error); extern void txn_runs_Writing_drop(TxnRunsWriting *self); extern void txn_runs_spill_directory(uint32 repository, uint32 own, StdPathBuf *result); extern bool txn_runs_cleaned(HostIoIo *io, HostError *error); extern bool txn_runs_checkpoint(HostIoIo *io, HostLsn redo, HostError *error); +extern bool txn_runs_save(HostIoIo *io, uint32 repository, uint32 own, bool *took, bool *cleaned, + HostError *error); extern bool txn_runs_loaded(HostIoIo *io, const TxnInsertPage *page, HostError *error); extern bool txn_runs_dropped(HostIoIo *io, uint32 repository, HostError *error); #ifdef DOLTGRES_TEST_TXN +extern void test_txn_runs_tests_a_starts_loaded_identity_pairs_count_at_their_writers_entry(void); +extern void test_txn_runs_tests_a_pair_the_save_folds_is_left_out_of_the_identity_run(void); +extern void test_txn_runs_tests_a_checkpoints_identity_run_loads_its_pairs_back(void); +extern void test_txn_runs_tests_an_identity_run_reads_its_pairs_and_refuses_heap_bytes_tuple_ids_and_other_ops(void); +extern void test_txn_runs_tests_a_helpers_part_linked_after_its_transactions_abort_leaves_with_its_files(void); +extern void test_txn_runs_tests_a_reader_that_listed_a_merges_source_reads_it_across_two_checkpoints(void); +extern void test_txn_runs_tests_an_index_storages_entries_spill_and_load_back_as_entries(void); extern void test_txn_runs_tests_a_checkpoint_names_a_running_spill_and_a_start_links_it_back(void); +extern void test_txn_runs_tests_a_deferred_spill_named_again_waits_for_two_new_sightings(void); extern void test_txn_runs_tests_a_spills_decoder_refuses_another_writer_and_a_wrong_index(void); +extern void test_txn_runs_tests_an_identity_spill_reads_its_pairs_and_refuses_another_writer_tuple_ids_and_heap_bytes(void); extern void test_txn_runs_tests_a_spills_file_reads_back_with_its_index(void); extern void test_txn_runs_tests_an_entries_runs_header_holds_its_table_and_index(void); extern void test_txn_runs_tests_an_index_storages_loaded_entries_stay_an_entries_run_through_a_second_checkpoint(void); diff --git a/src/include/doltgres/txn/spill.h b/src/include/doltgres/txn/spill.h index 66c7157fb0374..fca53d6f5d1a9 100644 --- a/src/include/doltgres/txn/spill.h +++ b/src/include/doltgres/txn/spill.h @@ -46,6 +46,9 @@ extern const TxnSpillHomeOps TXN_SPILL_STATE_HOME_OPS; /* TXN_SPILL_BRANCH_DELTAS_HOME_OPS is BranchDeltas' Home: a working state's delta, whose chunk tree spills. */ extern const TxnSpillHomeOps TXN_SPILL_BRANCH_DELTAS_HOME_OPS; +/* TXN_SPILL_IDENTITY_HOME_OPS is the identity spills' Home: a working state's delta, whose identity tree spills (bugs row 291). */ +extern const TxnSpillHomeOps TXN_SPILL_IDENTITY_HOME_OPS; + /* TxnSpillFilter is Filter: a spill's Bloom filter while it is built, in the backend's memory, its words of bits. */ typedef struct TxnSpillFilter { @@ -56,8 +59,10 @@ typedef struct TxnSpillFilter /* * TxnSpillFile is File: what names a spill's file, its spilling transaction's * top-level xid, its kind, its number among that transaction's files of its - * storage and tree, its length, its trailer's xxh3, and the count of its - * filter's bits. + * storage and tree, its length, its trailer's xxh3, the count of its filter's + * bits, for a free space map's spill its transaction's abort kept, the end of + * that abort, or for a carry's file of frozen versions its passing point, and, + * for a fold's residual of a map's spills, the fold's point. */ typedef struct TxnSpillFile { @@ -67,6 +72,20 @@ typedef struct TxnSpillFile uint64 len; uint64 hash; uint64 bits; + + /* + * passed is the end of the abort that kept a free space map's spill, or + * the newest end of a carry's frozen versions' records, 1 for versions + * with none, the point a fold passes it at, or 0 for none. + */ + uint64 passed; + + /* + * from is the point of the fold whose residual the file is, from which + * readers read it in its sources' place, or 0 for a file that is no + * residual. + */ + uint64 from; } TxnSpillFile; /* txn_TupleVecFileU64_t is listed's (Vec, u64): a storage's spill files, newest first, and the generation they were read at. */ @@ -90,6 +109,31 @@ typedef struct txn_TupleU64KindU32OptionU32_t uint32 f3; } txn_TupleU64KindU32OptionU32_t; +/* + * TxnSpillKept is Kept: a spilled version of a storage's map that a fold at + * a point leaves unapplied above an older running writer's (bugs row 228): + * its kind, its key, and its record's bytes. + */ +typedef struct TxnSpillKept +{ + TxnRunsKind kind; + StdVec key; + StdVec bytes; +} TxnSpillKept; + +/* + * TxnSpillChain is Chain: a key of a tree a carry writes to spill files: the + * tree, the key, and its versions oldest first, each its writer's top-level + * xid and its record. + */ +typedef struct TxnSpillChain +{ + TxnWrittenOf of; + StdVec key; + /* versions is a Vec<(u64, Vec)>. */ + StdVec versions; +} TxnSpillChain; + /* TXN_SPILL_PENDING_PROC_SLOTS_OPS is Pending's ProcSlots: each process's slot naming its one spill no manifest lists yet. */ extern const HostSharedProcSlotsOps TXN_SPILL_PENDING_PROC_SLOTS_OPS; @@ -103,32 +147,60 @@ extern bool txn_spill_replayed(HostIoIo *io, const HostStorage *storage, TxnWrit extern bool txn_spill_spilled(HostIoIo *io, const HostStorage *storage, TxnWrittenOf of, uint64 xid, bool *result, HostError *error); extern bool txn_spill_folded(HostIoIo *io, HostSharedRecord state, HostLsn point, HostError *error); +extern bool txn_spill_residual(HostIoIo *io, HostSharedRecord state, HostLsn point, const TxnSpillKept *kept, + size_t kept_len, StdVec *result, HostError *error); +extern bool txn_spill_unlisted(HostIoIo *io, HostSharedRecord state, const StdVec *residuals, HostError *error); +extern void txn_spill_Chain_drop(TxnSpillChain *self); + +struct TxnRecordCarryUnlisted; + +extern bool txn_spill_redo_unlisted(HostIoIo *io, const HostSharedHeld *held, + const struct TxnRecordCarryUnlisted *unlisted, HostError *error); +extern bool txn_spill_carried(HostIoIo *io, const HostSharedHeld *held, const uint64 *writers, size_t writers_len, + bool (*logged) (void *captures, HostSharedLongIo *io, StdVec *chains, + HostError *error), + void *captures, HostError *error); extern bool txn_spill_versions(const TxnSpillHomeOps *home, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, TxnRunsKind kind, const uint8 *key, size_t key_len, - StdVec *result, HostError *error); + uint64 *listed, StdVec *result, HostError *error); extern bool txn_spill_next(const TxnSpillHomeOps *home, HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, TxnRunsKind kind, bool has_after, const uint8 *after, size_t after_len, - TxnMemtableTableDirection way, bool *some, txn_TupleVecU8VecVersion_t *result, + TxnMemtableTableDirection way, uint64 *listed, bool *some, txn_TupleVecU8VecVersion_t *result, HostError *error); -extern bool txn_spill_holds(HostIoIo *io, HostSharedRecord state, TxnRunsKind kind, const uint8 *key, size_t key_len, - uint64 lsn, bool *result, HostError *error); +extern bool txn_spill_taken(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, + bool (*read) (void *captures, uint64 *listed, HostError *error), + void (*dropped) (void *captures), void *captures, HostError *error); +extern bool txn_spill_holds(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, TxnRunsKind kind, + const uint8 *key, size_t key_len, uint64 lsn, bool *result, HostError *error); extern uint64 txn_spill_generation(const TxnSpillHomeOps *home, HostSharedRecord state); extern bool txn_spill_newest_value(HostIoIo *io, HostSharedRecord state, uint32 id, bool *some, TxnMemtableMapEntry *result, HostError *error); extern bool txn_spill_settle_pending(HostIoIo *io, HostError *error); +extern bool txn_spill_take_settled(HostIoIo *io, HostError *error); extern bool txn_spill_wrote(HostIoIo *io, uint64 top, uint64 bytes, HostError *error); extern bool txn_spill_unwrote(HostIoIo *io, bool has_bytes, uint64 bytes, HostError *error); extern bool txn_spill_aborted(HostIoIo *io, uint64 xid, const HostStorage *storages, size_t storages_len, - HostError *error); + HostLsn end, HostError *error); +extern bool txn_spill_spilled_free(HostIoIo *io, uint64 xid, HostError *error); +extern bool txn_spill_redo_aborted(HostIoIo *io, uint64 xid, HostLsn end, HostError *error); extern bool txn_spill_chunks_repository(const HostStorage *storage, uint32 *result); extern bool txn_spill_Filter_of(size_t count, const StdBytes *keys, size_t keys_len, TxnSpillFilter *result, HostError *error); extern bool txn_spill_Filter_hits(const TxnSpillFilter *self, const uint8 *key, size_t key_len); extern HostStorage txn_spill_chunks_storage(uint32 repository); +extern HostStorage txn_spill_identity_storage(uint32 repository); +extern bool txn_spill_identity_applied(HostIoIo *io, uint64 xid, HostLsn point, bool *result, HostError *error); +extern bool txn_spill_identity_applies(HostIoIo *io, HostSharedRecord deltas, HostLsn point, bool *result, + HostError *error); +extern bool txn_spill_identity_folded(HostIoIo *io, HostSharedRecord deltas, HostLsn point, bool *took, + HostError *error); extern bool txn_spill_chunk_versions(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord deltas, const uint8 *key, size_t key_len, StdVec *result, HostError *error); extern bool txn_spill_chunk(HostIoIo *io, HostSharedRecord deltas, const uint8 *key, size_t key_len, bool *some, StdVec *result, HostError *error); +extern bool txn_spill_removed(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, + const HostStorage *storage, bool (*picked) (void *captures, const TxnSpillFile *file), + void *picked_captures, StdVec *result, HostError *error); extern bool txn_spill_linked(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord record, const HostStorage *storage, const TxnSpillFile *linking, size_t linking_len, bool has_replacing, txn_TupleU64KindU32OptionU32_t replacing, HostError *error); @@ -137,7 +209,7 @@ extern bool txn_spill_twinned(HostIoIo *io, HostSharedRecord state, TxnRunsKind extern uint64 txn_spill_counted(const TxnRunsDecoded *decoded); extern bool txn_spill_chunks_ended(HostIoIo *io, HostSharedRecord deltas, HostError *error); extern bool txn_spill_named(const TxnSpillHomeOps *home, HostIoIo *io, HostSharedRecord state, - const HostStorage *storage, StdVec *result, HostError *error); + const HostStorage *storage, HostLsn redo, StdVec *result, HostError *error); extern bool txn_spill_tuple_key(HostIoIo *io, const HostDeltaHold *hold, HostSharedRecord state, HostRow tid, bool *some, StdVec *result, HostError *error); @@ -145,6 +217,13 @@ extern bool txn_spill_tuple_key(HostIoIo *io, const HostDeltaHold *hold, HostSha extern void test_txn_spill_tests_a_merges_rewrite_and_another_transactions_listing_lose_nothing(void); extern void test_txn_spill_tests_a_reader_paused_in_a_manifest_read_finds_each_version_once(void); extern void test_txn_spill_tests_a_lookup_reads_an_evicted_filter_page_again(void); +extern void test_txn_spill_tests_a_spilled_free_space_maps_entries_are_read_below_the_memtable(void); +extern void test_txn_spill_tests_an_aborted_free_space_map_spill_stays_listed_until_a_fold(void); +extern void test_txn_spill_tests_the_committed_arm_spills_a_keys_oldest_committed_versions_above_its_spills(void); +extern void test_txn_spill_tests_the_committed_arms_spill_reads_back_in_order_after_a_restart(void); +extern void test_txn_spill_tests_a_free_space_spill_a_crash_ended_is_marked_with_the_next_checkpoints_redo_point(void); +extern void test_txn_spill_tests_a_prepared_transactions_spills_are_never_taken_out(void); +extern void test_txn_spill_tests_a_free_space_spills_merge_leaves_the_rows_tuple_ids_alone(void); extern void test_txn_spill_tests_a_transaction_spilling_more_storages_than_the_budget_holds_pages_completes(void); extern void test_txn_spill_tests_two_transactions_spilling_the_same_storages_crosswise_complete(void); extern void test_txn_spill_tests_more_transactions_than_any_fixed_count_spill_one_storage_at_a_full_budget(void); @@ -169,6 +248,9 @@ extern void test_txn_spill_tests_a_storages_spills_list_hit_and_remove(void); extern void test_txn_spill_tests_a_pending_spill_is_found_by_its_storages_readers_until_it_settles(void); extern void test_txn_spill_tests_a_filter_hits_its_keys_and_few_others(void); extern void test_txn_spill_tests_a_large_filter_takes_several_pages(void); +extern void test_txn_spill_tests_a_residual_whose_listing_fails_lists_nothing_and_gives_its_count_back(void); +extern void test_txn_spill_tests_a_drop_counts_a_crashed_transactions_files_in_either_order(void); +extern void test_txn_spill_tests_a_carry_that_fails_at_any_step_lists_nothing_and_leaves_every_count(void); #endif #endif /* DOLTGRES_TXN_SPILL_H */ diff --git a/src/include/doltgres/txn/state.h b/src/include/doltgres/txn/state.h index 72810f6607cff..d028c54c91960 100644 --- a/src/include/doltgres/txn/state.h +++ b/src/include/doltgres/txn/state.h @@ -74,6 +74,9 @@ extern const HostSharedPoolOps TXN_STATE_STATE_POOL_OPS; /* TXN_STATE_SPILLS_CHANGED_INDEX is SpillsChanged's atomic: the generation of the storage's spills' manifest. */ #define TXN_STATE_SPILLS_CHANGED_INDEX (22 + TXN_FREE_SPACE_CATEGORIES) +/* TXN_STATE_SPILLS_TAKEN_INDEX is SpillsTaken's atomic: the take generation of the storage's spills. */ +#define TXN_STATE_SPILLS_TAKEN_INDEX (31 + TXN_FREE_SPACE_CATEGORIES) + /* TXN_STATE_MAKERS_SINGLE_OPS is Makers' Shape and Single. */ extern const HostSharedSingleOps TXN_STATE_MAKERS_SINGLE_OPS; @@ -201,14 +204,28 @@ typedef struct txn_TupleU64HeldState_t HostSharedHeld f1; } txn_TupleU64HeldState_t; -/* TxnStateSaving is Saving: what a save's gathering stored in the states it gathered, its token and the states' locations. */ +/* + * TxnStateSaving is Saving: what a save's gathering stored in the states it + * gathered, its token and the states' locations, with the end the WAL is + * flushed through before the manifest names their bases. + */ typedef struct TxnStateSaving { uint64 token; /* locations is a Vec. */ StdVec locations; + HostLsn end; } TxnStateSaving; +/* + * TxnStateLogged is an install's logging, run before the new base is + * published, under the state's logging flag and no lock, with the flag's + * doors: it inserts the record that names the base, given the end of the + * record a standby's fold reached, and gives the end the WAL must reach + * before a manifest names the base, or zero. + */ +typedef bool (*TxnStateLogged) (void *captures, HostIoIoDoors io, HostLsn reached, HostLsn *end, HostError *error); + extern TxnBaseSizes txn_state_totals(const HostSharedHeld *held); extern bool txn_state_known(const HostSharedHeld *held); extern void txn_state_totals_unknown(HostSharedRecord record); @@ -254,18 +271,28 @@ extern bool txn_state_state(HostIoIo *io, const RepoRepo *repo, const HostShared const HostStorage *storage, TxnStatePersistence persistence, bool has_made, TxnStateMade *made, HostSharedHeld *result, HostError *error); extern bool txn_state_rechained(HostIoTakesShort io, const HostSharedHeld *held, uint64 read, - const TxnBaseBase *base, bool *result, HostError *error); + const TxnBaseBase *base, TxnStateLogged logged, void *logged_captures, bool *result, + HostError *error); extern bool txn_state_stored_base(HostIoIo *io, const RepoRepo *repo, const HostSharedReader *reading, const HostStorage *storage, bool *some, TxnBaseBase *result, HostError *error); extern bool txn_state_base_record(HostIoIo *io, const HostSharedHeld *held, bool *some, TxnBaseBase *result, HostError *error); extern bool txn_state_installed(HostIoTakesShort io, const HostSharedHeld *held, uint64 read, - const TxnBaseBase *base, bool *result, HostError *error); + const TxnBaseBase *base, TxnStateLogged logged, void *logged_captures, bool *result, + HostError *error); extern void txn_state_grown(HostSharedRecord record, const TxnBaseBase *base); extern bool txn_state_newest(HostIoIo *io, const HostSharedHeld *held, bool *some, TxnBaseBase *result, HostError *error); +extern bool txn_state_read_point(HostIoIo *io, HostSharedRecord record, uint64 *result, HostError *error); +extern bool txn_state_committed_taken(HostSharedRecord record, uint64 id); +extern void txn_state_committed_given(HostSharedRecord record, uint64 id); +extern uint32 txn_state_numbered(HostSharedRecord record, uint32 past); +extern uint64 txn_state_delta_bytes(HostSharedRecord record); extern bool txn_state_readable(HostIoIo *io, const HostSharedHeld *held, const HostSnapshot *snapshot, bool *some, TxnBaseBase *result, HostError *error); +extern bool txn_state_readable_since(HostIoIo *io, HostSharedRecord record, const HostSnapshot *snapshot, + uint64 generation, bool *moved, bool *some, TxnBaseBase *result, uint64 *now, + HostError *error); extern bool txn_state_emptied(HostIoIo *io, const HostSharedHeld *held, HostError *error); extern bool txn_state_redo_emptied(HostIoIo *io, const HostSharedHeld *held, HostError *error); extern bool txn_state_unlinked(HostIoIo *io, const HostStorage *storage, HostError *error); @@ -282,6 +309,7 @@ extern bool txn_state_held_to_end(HostIoIo *io, const HostStorage *storage, bool extern uint64 txn_state_counter(HostSharedRecord record); extern uint32 txn_state_space(HostSharedRecord record); extern uint64 txn_state_generation(HostSharedRecord record); +extern bool txn_state_moved_past(HostSharedRecord record, uint64 generation); extern void txn_state_moved(HostSharedRecord record, uint64 from, uint64 to); extern bool txn_state_spills_generation(uint64 *result, HostError *error); @@ -311,6 +339,7 @@ extern void test_txn_state_tests_an_install_keeps_a_key_recorded_after_its_fold_ extern void test_txn_state_tests_an_install_replaces_the_newest_and_refuses_a_moved_generation(void); extern void test_txn_state_tests_a_backend_records_its_key_in_a_state_and_redo_takes_none(void); extern void test_txn_state_tests_a_linked_free_space_map_or_tuple_id_table_keeps_the_counter_from_falling_back(void); +extern void test_txn_state_tests_a_state_with_spills_keeps_the_counter_from_falling_back(void); extern void test_txn_state_tests_a_save_gathers_a_linked_states_counter_and_lets_an_idle_ones_fall_back(void); extern void test_txn_state_tests_a_backend_holds_its_transactions_states_until_its_end(void); extern void test_txn_state_tests_a_state_saved_reclaimed_and_made_again_reads_the_same(void); @@ -323,6 +352,11 @@ extern void test_txn_state_tests_of_two_racing_fillers_the_second_to_the_lock_dr extern void test_txn_state_tests_a_state_is_filled_from_its_base_and_found_again(void); extern void test_txn_state_tests_the_largest_base_record_fits_a_states_blob_and_reads_back(void); extern void test_txn_state_tests_two_backends_that_open_one_storage_make_one_state(void); +extern void test_txn_state_tests_a_make_while_running_writers_hold_every_unsaved_state_gets_the_pools_error(void); +extern void test_txn_state_tests_a_rechain_keeps_a_key_recorded_while_it_logs(void); +extern bool txn_state_tests_reopened(HostIoIo *io, uint32 database, RepoRepo *result, HostError *error); +extern void txn_state_tests_saved(HostIoIo *io, const RepoRepo *repo, uint32 database, + void (*between) (void *captures, HostIoIo *io), void *between_captures); #endif #endif /* DOLTGRES_TXN_STATE_H */ diff --git a/src/include/doltgres/txn/status.h b/src/include/doltgres/txn/status.h index a798190849ca7..ea9fca0c6a830 100644 --- a/src/include/doltgres/txn/status.h +++ b/src/include/doltgres/txn/status.h @@ -137,7 +137,7 @@ extern bool txn_status_lookup(HostIoIo *io, HostXid xid, bool *has_result, TxnSt extern bool txn_status_take(HostIoIo *io, HostXid xid, TxnStatusOutcome outcome, TxnStatusRoomed *result_0, bool *result_1, HostError *error); extern bool txn_status_counted(HostIoTakesShort io, const HostDeltaHold *hold, TxnStatusRoomed roomed, - HostError *error); + uint64 amount, HostError *error); extern bool txn_status_settled(HostIoIo *io, HostXid xid, TxnStatusOutcome from, TxnStatusOutcome to, bool *result, HostError *error); extern bool txn_status_remove(HostIoIo *io, HostXid xid, bool *result, HostError *error); @@ -147,6 +147,8 @@ extern void txn_status_registered(const HostSharedListed *listed, HostXid xid, H extern void txn_status_prepared_registered(const HostSharedListed *listed, HostLsn end); extern bool txn_status_roomed(HostIoIo *io, HostXid top, TxnStatusRoomed *result, HostError *error); extern bool txn_status_prepared_committing(HostIoIo *io, HostXid xid, HostError *error); +extern bool txn_status_kept_roomed(HostIoIo *io, const TxnMemtableVersionWriter *writer, bool *has_result, + TxnStatusRoomed *result, HostError *error); extern bool txn_status_redo_roomed(HostIoIo *io, const TxnMemtableVersionWriter *writer, bool *has_result, TxnStatusRoomed *result, HostError *error); extern bool txn_status_redo_committed(HostIoIo *io, HostXid xid, HostLsn end, HostError *error); @@ -156,6 +158,21 @@ extern bool txn_status_Commit_Debug_fmt(const void *value, StdFormatter *f); extern bool txn_status_committed(HostIoIo *io, HostXid xid, TxnStatusCommit *result, HostError *error); #ifdef DOLTGRES_TEST_TXN +extern void test_txn_status_tests_a_prepared_commit_points_its_entry_at_the_committer_and_registers(void); +extern void test_txn_status_tests_a_slot_stays_its_entrys_across_a_growth_and_a_removed_ones_is_taken_again(void); +extern void test_txn_status_tests_a_growth_while_the_proc_array_cant_answer_keeps_an_ended_entry(void); +extern void test_txn_status_tests_a_commit_registered_in_its_slot_reads_back_and_its_move_settles_the_entry(void); +extern void test_txn_status_tests_a_folds_point_lies_below_a_commit_still_in_the_proc_array(void); +extern void test_txn_status_tests_a_take_of_room_moves_the_slots_pair_first(void); +extern void test_txn_status_tests_a_replayed_commit_stores_its_end_in_the_entry_its_replay_took(void); +extern void test_txn_status_tests_churn_with_no_live_entries_keeps_one_segment(void); +extern void test_txn_status_tests_a_growth_holds_what_it_keeps_in_the_least_room(void); +extern void test_txn_status_tests_entries_are_taken_found_counted_removed_and_kept_by_a_growth(void); +extern void test_txn_status_tests_an_outcomes_word_says_the_outcome(void); +extern void test_txn_status_tests_an_outcome_too_large_for_its_word_has_none(void); +extern void test_txn_status_tests_an_xids_word_is_no_empty_places(void); +extern void test_txn_status_tests_a_lookup_probes_every_place_and_neighbours_start_apart(void); +extern void test_txn_status_tests_a_table_is_crowded_at_three_quarters(void); extern void test_txn_status_tests_a_key_whose_slot_isnt_stored_yet_leaves_its_slot_as_it_was(void); #endif diff --git a/src/include/doltgres/txn/vacuum.h b/src/include/doltgres/txn/vacuum.h index 273007589f41a..4709b654f4489 100644 --- a/src/include/doltgres/txn/vacuum.h +++ b/src/include/doltgres/txn/vacuum.h @@ -28,6 +28,7 @@ #include "doltgres/trees/store.h" #include "doltgres/txn/base.h" #include "doltgres/txn/insert.h" +#include "doltgres/txn/memtable/version.h" #include "doltgres/txn/record.h" /* @@ -38,8 +39,9 @@ /* * TxnVacuumFreed is Freed: a tuple id never_held answers, its heap bytes, - * its free's conflict horizon, and the value ids the free takes down, a - * Vec. + * its free's conflict horizon, the value ids the free takes down, a + * Vec, and its row's stored ender, as the pending tree stores it, which + * its record names (bugs row 320). */ typedef struct TxnVacuumFreed { @@ -47,6 +49,7 @@ typedef struct TxnVacuumFreed uint64 f1; uint64 f2; StdVec f3; + uint64 f4; } TxnVacuumFreed; /* @@ -62,6 +65,26 @@ struct TxnVacuumPrune extern bool txn_vacuum_never_held(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, const TxnInsertPage *page, const HostSnapshot *horizon, StdVec *result, HostError *error); + +/* + * TxnVacuumLeaving is a take a fold's drop takes out of the delta, as + * VACUUM's find reads it: its tuple id, its take's writer, the bytes its take + * lowered, its stored ender, and its row's committed value ids. + */ +typedef struct TxnVacuumLeaving +{ + uint64 tid; + TxnMemtableVersionWriter writer; + uint64 bytes; + uint64 ender; + /* ids is a Vec. */ + StdVec ids; +} TxnVacuumLeaving; + +extern bool txn_vacuum_leaving(HostIoIo *io, TreesStoreNodeStore store, const TxnBaseBase *base, + const TxnInsertPage *page, TxnMemtableVersionVersion *run, size_t run_len, + StdVec *result, HostError *error); +extern void txn_vacuum_leaving_drop(StdVec *leaving); extern bool txn_vacuum_frees(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, const TxnBaseBase *base, uint64 tid, bool *result, HostError *error); extern bool txn_vacuum_spent(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *held, uint32 block, @@ -80,4 +103,44 @@ extern bool txn_vacuum_redo_freed(HostIoIo *io, TreesStoreNodeStore store, const extern bool txn_vacuum_taken_out(HostIoIo *io, const HostSharedHeld *held, const TxnRecordTidsFreed *freed, uint64 lsn, HostError *error); +#ifdef DOLTGRES_TEST_TXN +#include "doltgres/simc/simc.h" + +extern SimcTaken txn_vacuum_tests_every(void); +extern bool txn_vacuum_tests_all_returned(const SimcReport *report); +extern void txn_vacuum_tests_returned(const SimcReport *report, const char *file, uint32 ended_line, + uint32 ended_col, uint32 line, uint32 col); +extern void test_txn_vacuum_tests_an_aborted_inserts_tuple_id_on_the_fresh_page_is_freed_with_its_own_bytes_and_taken_again(void); +extern void test_txn_vacuum_tests_a_take_that_empties_its_entry_is_freed_with_all_the_bytes_the_entry_held(void); +extern void test_txn_vacuum_tests_a_take_no_row_names_is_freed_only_once_its_taker_has_ended(void); +extern void test_txn_vacuum_tests_an_aborted_leaves_old_offsets_and_its_new_tuple_id_are_handed_out_again(void); +extern void test_txn_vacuum_tests_a_skipped_index_pass_gives_back_a_dead_tuple_ids_bytes_and_a_later_free_its_offset_alone(void); +extern void test_txn_vacuum_tests_a_dead_tuple_id_on_a_full_block_keeps_its_bytes_until_its_free(void); +extern void test_txn_vacuum_tests_a_take_prunes_a_full_blocks_dead_versions_and_its_update_stays_on_the_block(void); +extern void test_txn_vacuum_tests_a_take_leaves_a_version_an_open_snapshot_may_see(void); +extern void test_txn_vacuum_tests_a_dead_tuple_id_is_claimed_once(void); +extern void test_txn_vacuum_tests_a_refused_markers_redo_keeps_the_version_as_the_primary_does(void); +extern void test_txn_vacuum_tests_every_raise_of_a_blocks_room_lowers_its_categorys_hint(void); +extern void test_txn_vacuum_tests_a_folded_rows_tuple_id_is_not_freed(void); +extern void test_txn_vacuum_tests_a_fold_between_the_finds_reads_leaves_its_rows_held(void); +extern void test_txn_vacuum_tests_a_fold_between_a_prunes_reads_leaves_what_a_prune_after_it_leaves(void); +extern void test_txn_vacuum_tests_a_fold_during_a_blocks_spent_count_counts_what_a_count_after_it_counts(void); +extern void test_txn_vacuum_tests_a_fold_during_a_find_finds_what_a_find_after_it_finds(void); +extern void test_txn_vacuum_tests_a_fold_during_a_free_leaves_what_a_free_after_it_leaves(void); +extern void test_txn_vacuum_tests_a_fold_during_a_frees_redo_leaves_what_the_free_left(void); +extern void test_txn_vacuum_tests_a_fold_during_a_frees_redo_leaves_what_a_redo_after_it_leaves(void); +extern void test_txn_vacuum_tests_a_dropped_rows_tuple_id_keeps_its_enders_xid_for_its_frees_horizon(void); +extern void test_txn_vacuum_tests_a_free_of_a_block_whose_takes_all_spilled_starts_from_its_newest_spilled_entry(void); +extern void test_txn_vacuum_tests_a_takes_ender_widens_against_its_writer_across_an_epoch(void); +extern void test_txn_vacuum_tests_a_dropped_rows_tuple_id_waits_for_the_horizon_to_pass_its_ender(void); +extern void test_txn_vacuum_tests_a_row_its_own_transaction_inserted_and_deleted_waits_for_the_horizon(void); +extern void test_txn_vacuum_tests_a_drops_ender_goes_on_its_rows_take_not_a_later_take_of_the_tuple_id(void); +extern void test_txn_vacuum_tests_a_frees_take_out_leaves_a_retake_on_its_key(void); +extern void test_txn_vacuum_tests_a_frees_take_out_with_no_ender_leaves_any_retake_on_its_key(void); +extern void test_txn_vacuum_tests_a_frees_take_out_takes_out_a_subtransactions_delete(void); +extern void test_txn_vacuum_tests_a_frees_enders_of_another_form_are_refused_as_corruption(void); +extern void test_txn_vacuum_tests_a_prunes_take_out_of_an_aborted_writers_own_row_leaves_a_retake(void); +extern void test_txn_vacuum_tests_a_prune_over_an_aborted_unfolded_insert_claims_nothing(void); +#endif + #endif /* DOLTGRES_TXN_VACUUM_H */ diff --git a/src/include/doltgres/txn/values.h b/src/include/doltgres/txn/values.h index b81129f90d3c9..a746f3f3fa077 100644 --- a/src/include/doltgres/txn/values.h +++ b/src/include/doltgres/txn/values.h @@ -79,4 +79,12 @@ extern TxnMemtableMapStep txn_values_taken_down(bool has_current, const uint8 *c extern bool txn_values_based(HostIoIo *io, TreesStoreNodeStore store, const HostSharedHeld *toast, uint32 id, bool *has_value, StdVec *value, std_TupleU64U64_t *mark, HostError *error); +#ifdef DOLTGRES_TEST_TXN +extern void test_txn_values_tests_a_value_map_entry_reads_back_and_refuses_every_other_form(void); +extern void test_txn_values_tests_a_value_id_the_map_holds_is_skipped(void); +extern void test_txn_values_tests_a_value_ids_entry_reads_back(void); +extern void test_txn_values_tests_a_value_counts_heaps_toast_chunks(void); +extern void test_txn_values_tests_a_take_down_takes_one_from_a_count_and_removes_the_last(void); +#endif + #endif /* DOLTGRES_TXN_VALUES_H */ diff --git a/src/include/doltgres/txn/visible.h b/src/include/doltgres/txn/visible.h index 38538c44dfd1f..e60c25d051a9c 100644 --- a/src/include/doltgres/txn/visible.h +++ b/src/include/doltgres/txn/visible.h @@ -282,6 +282,20 @@ extern bool txn_visible_carries(HostIoIo *io, TxnMemtableVersionVersion *version HostRow tuple, uint32 command, bool *has, uint64 *result, HostError *error); #ifdef DOLTGRES_TEST_TXN +extern void test_txn_visible_tests_a_rows_change_is_answered_as_heaps_satisfies_update_answers(void); +extern void test_txn_visible_tests_a_killed_speculative_insert_is_aborted_for_every_reader_at_once(void); +extern void test_txn_visible_tests_a_rows_slice_answers_as_its_whole_chain(void); +extern void test_txn_visible_tests_a_rows_slice_after_a_removal_under_its_head_answers_as_its_whole_chain(void); +extern void test_txn_visible_tests_a_versions_epoch_counts_the_transforms_written_before_it(void); +extern void test_txn_visible_tests_each_snapshot_kind_sees_the_versions_heaps_rule_sees(void); +extern void test_txn_visible_tests_a_key_read_gives_every_row_version_the_snapshots_rule_sees(void); +extern void test_txn_visible_tests_a_row_named_by_its_tuple_id_is_ended_by_the_version_that_names_it(void); +extern void test_txn_visible_tests_a_rows_stamp_shows_the_last_change_above_it_whatever_its_outcome(void); +extern void test_txn_visible_tests_a_historic_snapshot_sees_its_transactions_rows_by_their_commands(void); +extern void test_txn_visible_tests_a_serializable_read_conflicts_out_with_the_writers_heaps_rule_names(void); +extern void test_txn_visible_tests_a_transactions_own_row_is_seen_from_the_command_after_its_write(void); +extern void test_txn_visible_tests_a_subtransactions_row_stays_invisible_to_a_snapshot_taken_while_its_transaction_ran(void); +extern void test_txn_visible_tests_a_rows_delete_is_seen_only_by_a_snapshot_taken_after_its_commit(void); extern void test_txn_visible_tests_a_running_writers_version_leaves_its_ran_answer_as_it_was(void); #endif diff --git a/src/include/doltgres/txn/written.h b/src/include/doltgres/txn/written.h index 73cf77112d676..9c2294be94c5b 100644 --- a/src/include/doltgres/txn/written.h +++ b/src/include/doltgres/txn/written.h @@ -30,11 +30,17 @@ /* TXN_WRITTEN_CAP is CAP, the most keys the list keeps; past it the list stops growing and is marked overflowed. */ #define TXN_WRITTEN_CAP ((size_t) 64 * 1024) -/* TxnWrittenOf is Of: the tree a touched key is in, a storage's rows, its value map, its database's identity tree, or its repository's chunk tree. */ +/* + * TxnWrittenOf is Of: the tree a touched key is in, a storage's rows, its + * value map, its free space map, its database's identity tree, or its + * repository's chunk tree. A free space map's take isn't taken back, so only + * its bytes are counted, for its spill. + */ typedef enum TxnWrittenOf { TXN_WRITTEN_OF_ROWS, TXN_WRITTEN_OF_VALUES, + TXN_WRITTEN_OF_FREE, TXN_WRITTEN_OF_IDENTITY, TXN_WRITTEN_OF_CHUNKS, } TxnWrittenOf; @@ -104,7 +110,7 @@ extern bool txn_written_spilled(HostIoIo *io, const HostStorage *storage, TxnWri extern bool txn_written_keys(HostIoIo *io, bool *some, txn_TupleU64VecTupleStorageOfVecVecU8_t *result, HostError *error); extern bool txn_written_ended(HostIoIo *io, HostError *error); -extern bool txn_written_aborted(HostIoIo *io, HostError *error); +extern bool txn_written_aborted(HostIoIo *io, uint64 end, HostError *error); extern bool txn_written_sub_started(HostIoIo *io, uint32 sub, HostError *error); extern bool txn_written_sub_committed(HostIoIo *io, uint32 sub, HostError *error); extern bool txn_written_sub_aborted(HostIoIo *io, uint32 sub, HostError *error); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index da2b9199c4543..86e4841121a54 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -946,6 +946,7 @@ DoltgresEntriesOp DoltgresEntriesOpenedStep DoltgresEntriesOthers DoltgresEntriesOthersTag +DoltgresEntriesScanned DoltgresEntriesSearchesEntries DoltgresEntriesSearchesKeyed DoltgresEntriesSearchesRange @@ -1004,7 +1005,6 @@ DoltgresRekeyMoved DoltgresRekeyRekeyed DoltgresRekeyVisible DoltgresRelationCreating -DoltgresRewriteCarry DoltgresRewriteCounts DoltgresRewriteFrom DoltgresRewriteFromTag @@ -1025,6 +1025,7 @@ DoltgresScanEpochs DoltgresScanFound DoltgresScanGroup DoltgresScanLayout +DoltgresScanNewest DoltgresScanPage DoltgresScanPageTag DoltgresScanPaired @@ -1033,6 +1034,8 @@ DoltgresScanPicked DoltgresScanPickedTag DoltgresScanPlace DoltgresScanPlaceTag +DoltgresScanRebased +DoltgresScanResolved DoltgresScanRow DoltgresScanRun DoltgresScanSampled @@ -1048,6 +1051,7 @@ DoltgresScanTurn DoltgresScanTurnTag DoltgresScanUnder DoltgresScanUnit +DoltgresScanUnitPlacing DoltgresSettleOp DoltgresSettleSettling DomainConstraintCache @@ -1726,6 +1730,7 @@ HostLockIoMutexHeld HostLockIoMutexHome HostLockIoMutexOpen HostLockIoMutexRepoCopy +HostLockIoMutexVecU64 HostLockIoRwLock HostLockLending HostLockLockedRwLockReadGuard @@ -3024,6 +3029,7 @@ PgZstdTestsResultVecU8Failed Pg_abi_values Pg_finfo_record Pg_magic_struct +PghostDeltaOpen PghostDeltaPages PghostDeltaRecoveryFold PghostPgHost @@ -3596,6 +3602,7 @@ ShmemRecordsOp ShmemRecordsOpTag ShmemRecordsPlaced ShmemRecordsPlan +ShmemRecordsResolved ShmemRecordsSpot ShmemRegion ShmemRequest @@ -3613,6 +3620,7 @@ SimcEnding SimcEndingTag SimcOutcome SimcOutcomeTag +SimcProcess SimcReport SimcRun SimcSim @@ -4336,6 +4344,7 @@ TxnFoldCaught TxnFoldCommitting TxnFoldCopies TxnFoldCtidEdit +TxnFoldEach TxnFoldFits TxnFoldFoldedRows TxnFoldFolding @@ -4350,6 +4359,7 @@ TxnFoldReformOps TxnFoldReformRow TxnFoldReformed TxnFoldSpaced +TxnFoldTakenOut TxnFoldTrees TxnFreeSpaceFree TxnIdentityDatabase @@ -4361,6 +4371,8 @@ TxnIdentityKind TxnIdentityPair TxnIdentityPaired TxnInsertBase +TxnInsertCarriedPut +TxnInsertCarriedTake TxnInsertCompleted TxnInsertEqual TxnInsertEqualOps @@ -4404,6 +4416,7 @@ TxnMemtableIndexPlace TxnMemtableLockMode TxnMemtableMapEntry TxnMemtableMapMap +TxnMemtableMapRead TxnMemtableMapStep TxnMemtableMapTake TxnMemtableMapTaken @@ -4438,14 +4451,17 @@ TxnMemtableTableOnePage TxnMemtableTableOwnerOps TxnMemtableTablePreparedBranchDeltas TxnMemtableTablePreparedState +TxnMemtableTableRefusal TxnMemtableTableRemoving TxnMemtableTableReservation TxnMemtableTableReserve TxnMemtableTableRetiring TxnMemtableTableTarget TxnMemtableTableTreeOps +TxnMemtableTuplesLive TxnMemtableTuplesRetiring TxnMemtableTuplesTaken +TxnMemtableTuplesTestsRefusal TxnMemtableTuplesWant TxnMemtableTuplesWantTag TxnMemtableVersionAddresses @@ -4474,6 +4490,7 @@ TxnRecordAdvancedParts TxnRecordBaseAdvanced TxnRecordBuiltNode TxnRecordCarriedTake +TxnRecordCarryUnlisted TxnRecordChunkWrite TxnRecordCompletion TxnRecordDatabaseChange @@ -4487,16 +4504,23 @@ TxnRecordRead TxnRecordRowChange TxnRecordSpent TxnRecordStorageEmptied +TxnRecordTestsExtra +TxnRecordTestsField +TxnRecordTestsRead +TxnRecordTestsRuns +TxnRecordTestsU32s TxnRecordTidsFreed TxnRecordToast TxnRecordTree TxnRewriteBased TxnRewriteBlock +TxnRewriteCarry TxnRewriteCopied TxnRewriteKeysEach TxnRewritePacked TxnRewriteStored TxnRewriteStoredTag +TxnRewriteTheirs TxnRootsCaptured TxnRunsDecoded TxnRunsEncoder @@ -4509,15 +4533,24 @@ TxnRunsRemoval TxnRunsTail TxnRunsTailTag TxnRunsWriting +TxnSpillChain +TxnSpillCommittedState TxnSpillFile +TxnSpillFilesResidual TxnSpillFilter TxnSpillFound TxnSpillFoundTag +TxnSpillGroup TxnSpillHomeOps +TxnSpillKept +TxnSpillLayerFile +TxnSpillLeft TxnSpillNaming +TxnSpillOldest TxnSpillPended TxnSpillRead TxnSpillReading +TxnSpillSlot TxnSpillSource TxnSpillTurn TxnSpillTurnSlotTag @@ -4527,6 +4560,7 @@ TxnStateFindTag TxnStateFound TxnStateFoundTag TxnStateHolds +TxnStateLogged TxnStateMade TxnStateMissing TxnStateMove @@ -4550,6 +4584,7 @@ TxnStatusTook TxnVacuumDown TxnVacuumFound TxnVacuumFreed +TxnVacuumLeaving TxnVacuumPending TxnVacuumPrune TxnVacuumRemoval @@ -4568,6 +4603,16 @@ TxnVisibleRuleOps TxnVisibleSeen TxnVisibleSeenRow TxnVisibleSerial +TxnVisibleTestsAnswer +TxnVisibleTestsCase +TxnVisibleTestsChanged +TxnVisibleTestsConflict +TxnVisibleTestsFound +TxnVisibleTestsKeyed +TxnVisibleTestsRead +TxnVisibleTestsShown +TxnVisibleTestsWrite +TxnVisibleTestsXids TxnVisibleUpdatable TxnVisibleUpdatableTag TxnVisibleVacuumed @@ -5171,6 +5216,7 @@ chunks_packs_backend_Closing_frame_0_t chunks_packs_backend_Packs_changed_frame_0_t chunks_packs_backend_Packs_compact_frame_0_t chunks_packs_backend_Packs_copied_frame_0_t +chunks_packs_backend_Packs_flush_frame_0_t chunks_packs_backend_Packs_forward_frame_0_t chunks_packs_backend_Packs_forward_moves_t chunks_packs_backend_Packs_listed_frame_0_t @@ -5389,6 +5435,7 @@ doltgres_TupleVisibleBool_t doltgres_TupleVisibleChangeOptionU64_t doltgres_deletes_Deleting_new_frame_t doltgres_deletes_sort_and_shrink_frame_t +doltgres_entries_BuildSnapshot doltgres_entries_Building doltgres_entries_Built doltgres_entries_Changing @@ -5410,6 +5457,7 @@ doltgres_entries_bitmap_t doltgres_entries_blob_bytes_frame_t doltgres_entries_build_read_captures_t doltgres_entries_build_scanned_frame_t +doltgres_entries_build_snapshot_taken_t doltgres_entries_building_replaced_t doltgres_entries_builtin_frame_0_t doltgres_entries_chained_captures_t @@ -5427,6 +5475,7 @@ doltgres_entries_doltgres_database_size_frame_t doltgres_entries_doltgres_index_blocks_frame_t doltgres_entries_doltgres_index_build_callback_frame_t doltgres_entries_doltgres_index_build_frame_t +doltgres_entries_doltgres_index_build_range_scan_captures_t doltgres_entries_doltgres_index_build_range_scan_frame_t doltgres_entries_doltgres_index_buildempty_frame_t doltgres_entries_doltgres_index_copy_data_captures_t @@ -5488,6 +5537,7 @@ doltgres_entries_doltgres_rmgr_desc_frame_t doltgres_entries_doltgres_rmgr_redo_frame_t doltgres_entries_doltgres_scan_begin_captures_t doltgres_entries_doltgres_scan_begin_frame_t +doltgres_entries_doltgres_tablespace_emptied_frame_t doltgres_entries_doltgres_tuple_fetch_row_version_frame_t doltgres_entries_entry_record_frame_t doltgres_entries_entry_unique_captures_t @@ -5507,6 +5557,7 @@ doltgres_entries_given_frame_t doltgres_entries_keeping_frame_t doltgres_entries_libc_locale_frame_0_t doltgres_entries_loaded_frame_t +doltgres_entries_loaded_logging_t doltgres_entries_locked_frame_t doltgres_entries_moved_into_captures_t doltgres_entries_moved_into_closure_0_frame_t @@ -5537,6 +5588,7 @@ doltgres_entries_rows_closure_0_frame_t doltgres_entries_rows_frame_t doltgres_entries_running_replayed_captures_t doltgres_entries_running_replayed_frame_t +doltgres_entries_scan_frame_t doltgres_entries_searches_Entries_decoded_frame_0_t doltgres_entries_searches_Entries_fields_frame_0_t doltgres_entries_searches_Entries_ordered_frame_0_t @@ -5551,6 +5603,7 @@ doltgres_entries_searches_bitmap_frame_0_t doltgres_entries_searches_current_frame_0_t doltgres_entries_searches_next_frame_0_t doltgres_entries_searches_preprocessed_frame_0_t +doltgres_entries_searches_read_captures_t doltgres_entries_searches_read_frame_0_t doltgres_entries_searches_restore_frame_0_t doltgres_entries_sample_next_block_t @@ -5575,6 +5628,7 @@ doltgres_entries_testing_file_step_frame_0_t doltgres_entries_testing_find_frame_0_t doltgres_entries_testing_held_frame_0_t doltgres_entries_testing_identity_frame_0_t +doltgres_entries_testing_identity_versions_frame_0_t doltgres_entries_testing_inner_frame_0_t doltgres_entries_testing_joined_frame_0_t doltgres_entries_testing_key_frame_0_t @@ -5672,6 +5726,7 @@ doltgres_rewrite_rewritten_frame_t doltgres_rewrite_row_ends_t doltgres_rewrite_store_frame_t doltgres_scan_Epochs_transform_frame_t +doltgres_scan_Kept_t doltgres_scan_OptionBounds_t doltgres_scan_OptionChosen_t doltgres_scan_OptionCopies_t @@ -5680,6 +5735,7 @@ doltgres_scan_OptionRow_t doltgres_scan_OptionTupleU32U16_t doltgres_scan_OptionTupleU64U64_t doltgres_scan_OptionU64_t +doltgres_scan_Trees_newer_frame_t doltgres_scan_TupleOptionTreeOptionRun_t doltgres_scan_TupleOptionVecU8OptionVecU8_t doltgres_scan_TupleOptionVecU8VecVecU8_t @@ -5713,6 +5769,11 @@ doltgres_scan_pick_frame_t doltgres_scan_place_read_t doltgres_scan_placed_captures_t doltgres_scan_placed_frame_t +doltgres_scan_rebase_read_t +doltgres_scan_rebased_frame_t +doltgres_scan_rebased_put_t +doltgres_scan_rebasing_frame_t +doltgres_scan_reseated_frame_t doltgres_scan_row_frame_t doltgres_scan_run_frame_t doltgres_scan_run_set_t @@ -5721,9 +5782,12 @@ doltgres_scan_seek_compared_frame_t doltgres_scan_seek_frame_t doltgres_scan_seek_put_t doltgres_scan_seek_t +doltgres_scan_sought_captures_t +doltgres_scan_sought_read_t doltgres_scan_split_keys_frame_t doltgres_scan_splits_captures_t doltgres_scan_tests_case_t +doltgres_scan_tests_held_t doltgres_scan_tests_keys_t doltgres_scan_tests_pair_t doltgres_scan_tests_shown_t @@ -5731,6 +5795,7 @@ doltgres_scan_tests_under_t doltgres_scan_tid_valid_captures_t doltgres_scan_turn_t doltgres_scan_unit_keys_frame_t +doltgres_scan_unit_placed_frame_t doltgres_scan_written_frame_t doltgres_settle_closed_captures_t doltgres_settle_held_captures_t @@ -5886,6 +5951,7 @@ host_cache_PINS_t host_cache_hash host_delta_With_own_t host_delta_With_pages_t +host_delta_directory_tests_Directory_t hstoreCheckKeyLen_t hstoreCheckValLen_t hstorePairs_t @@ -6413,8 +6479,10 @@ sigjmp_buf signedbitmapword sigset_t simc_TupleSqlstateString_t +simc_TupleStorageRowNewCidLsn_t simc_TupleStringEnding_t simc_TupleStringSqlstateString_t +simc_TupleU8VecU8Lsn_t size_t slist_head slist_iter @@ -6731,10 +6799,11 @@ txn_TupleBoolU32U32BoolBoolU8_t txn_TupleEntryU64_t txn_TupleFileU64_t txn_TupleHandleOptionU64_t +txn_TupleHandleU64Bool_t txn_TupleHandleU64_t -txn_TupleHandleWriter_t txn_TupleLocatorLocator_t -txn_TuplePathBufU64_t +txn_TuplePathBufU64Bool_t +txn_TupleRefSliceU8RefSliceU8U_t txn_TupleRefSliceU8U64_t txn_TupleRefSliceU8Usize_t txn_TupleRowXid_t @@ -6744,7 +6813,7 @@ txn_TupleStorageOf_t txn_TupleStorageU64VecFile_t txn_TupleU32Entry_t txn_TupleU32TupleEntryU64_t -txn_TupleU32VecTupleHandleWriter_t +txn_TupleU32VecHandle_t txn_TupleU64Bool_t txn_TupleU64HeldState_t txn_TupleU64KindU32OptionU32_t @@ -6765,7 +6834,6 @@ txn_TupleVecU32Bool_t txn_TupleVecU8Op_t txn_TupleVecU8U64_t txn_TupleVecU8VecHandle_t -txn_TupleVecU8VecTupleHandleOptionU64_t txn_TupleVecU8VecU8_t txn_TupleVecU8VecVecU8_t txn_TupleVecU8VecVersion_t @@ -6781,6 +6849,8 @@ txn_blobs_Blobs_read_frame_0_t txn_blobs_kept_frame_0_t txn_blobs_redo_closure_captures_t txn_blobs_redo_frame_0_t +txn_blobs_tests_holding_t +txn_blobs_tests_logged_t txn_blobs_written_closure_captures_t txn_blobs_written_closure_frame_0_t txn_blobs_written_frame_0_t @@ -6788,6 +6858,9 @@ txn_change_end_closure_0_captures_t txn_change_end_closure_0_frame_t txn_change_end_frame_t txn_change_redo_closure_0_captures_t +txn_change_tests_logged_t +txn_change_tests_made_t +txn_change_tests_shown_t txn_entry_TupleRowChangeUsize_t txn_entry_TupleU64Bool_t txn_entry_base_stamps_closure_1_captures_t @@ -6799,6 +6872,10 @@ txn_entry_copies_frame_t txn_entry_encodings_t txn_entry_formed_frame_t txn_entry_pushed_frame_t +txn_entry_tests_logged_t +txn_entry_tests_made_t +txn_entry_tests_shown_t +txn_entry_tests_unique_t txn_entry_unique_closure_2_captures_t txn_entry_unique_closure_2_frame_t txn_entry_unique_frame_t @@ -6820,6 +6897,7 @@ txn_fold_folded_frame_t txn_fold_folded_key_closure_0_captures_t txn_fold_folded_key_closure_0_frame_t txn_fold_folded_key_frame_t +txn_fold_folded_logging_t txn_fold_folded_rows_frame_t txn_fold_held_pairing_t txn_fold_in_order_frame_t @@ -6845,6 +6923,7 @@ txn_fold_valued_frame_t txn_fold_valued_sizes_frame_t txn_fold_whole_closure_0_frame_t txn_fold_whole_frame_t +txn_fold_whole_logging_t txn_free_space_seek_captures_t txn_identity_changed_closure_4_frame_t txn_identity_changed_frame_t @@ -6862,6 +6941,8 @@ txn_identity_paired_frame_t txn_identity_redo_closure_0_captures_t txn_identity_redo_frame_t txn_identity_removed_frame_t +txn_identity_tests_opened_t +txn_identity_tests_writing_t txn_insert_JumpStep_t txn_insert_ReplayStep_t txn_insert_TakeStep_t @@ -6872,6 +6953,8 @@ txn_insert_based_closure_2_captures_t txn_insert_based_closure_2_frame_t txn_insert_based_frame_t txn_insert_candidate_frame_t +txn_insert_carried_entry_t +txn_insert_carried_frame_t txn_insert_complete_frame_t txn_insert_jump_closure_0_captures_t txn_insert_jump_frame_t @@ -6881,6 +6964,23 @@ txn_insert_redo_spent_closure_0_captures_t txn_insert_redo_spent_frame_t txn_insert_resolving_frame_t txn_insert_seen_frame_t +txn_insert_tests_chain_t +txn_insert_tests_found_t +txn_insert_tests_free_t +txn_insert_tests_from_t +txn_insert_tests_kept_t +txn_insert_tests_last_page_t +txn_insert_tests_left_t +txn_insert_tests_loaded_t +txn_insert_tests_logged_t +txn_insert_tests_logged_tag_t +txn_insert_tests_mapped_t +txn_insert_tests_op_t +txn_insert_tests_replica_t +txn_insert_tests_resolving_t +txn_insert_tests_shown_t +txn_insert_tests_straddle_t +txn_insert_tests_valued_t txn_insert_turn_closure_9_frame_t txn_insert_turn_frame_t txn_insert_written_frame_t @@ -6907,6 +7007,7 @@ txn_memtable_index_built_frame_0_t txn_memtable_index_copied_frame_0_t txn_memtable_index_moved_frame_0_t txn_memtable_map_cut_reading_t +txn_memtable_map_marked_read_t txn_memtable_map_named_t txn_memtable_map_newest_reading_t txn_memtable_map_rows_reading_t @@ -6920,6 +7021,7 @@ txn_memtable_map_tests_case_t txn_memtable_map_tests_map_only_t txn_memtable_map_tests_newest_t txn_memtable_map_tests_newests_t +txn_memtable_map_tests_read_t txn_memtable_map_tests_sought_list_t txn_memtable_map_tests_sought_t txn_memtable_map_tests_taken_frame_0_t @@ -6956,6 +7058,7 @@ txn_memtable_table_held_t txn_memtable_table_nearest_frame_0_t txn_memtable_table_nearest_t txn_memtable_table_newest_t +txn_memtable_table_next_read_t txn_memtable_table_prepared_frame_0_t txn_memtable_table_pruned_frame_0_t txn_memtable_table_pruned_t @@ -6994,10 +7097,12 @@ txn_memtable_table_tests_walker_t txn_memtable_table_tests_windowed_frame_0_t txn_memtable_table_tests_written_frame_0_t txn_memtable_table_unlisted_frame_0_t +txn_memtable_table_versions_read_t txn_memtable_table_walk_t txn_memtable_table_within_t txn_memtable_testing_pushed_frame_0_t txn_memtable_testing_ranged_frame_0_t +txn_memtable_tuples_compacted_frame_t txn_memtable_tuples_finish_frame_0_t txn_memtable_tuples_first_frame_0_t txn_memtable_tuples_grown_frame_0_t @@ -7006,11 +7111,13 @@ txn_memtable_tuples_pushed_frame_0_t txn_memtable_tuples_reserve_frame_0_t txn_memtable_tuples_tests_counts_frame_0_t txn_memtable_tuples_tests_counts_t +txn_memtable_tuples_tests_insert_t txn_memtable_tuples_tests_stored_frame_0_t txn_memtable_tuples_tests_stored_t txn_memtable_version_owned_frame_t txn_memtable_version_removed_frame_t txn_memtable_version_stored_frame_t +txn_memtable_version_tests_read_t txn_record_advanced_frame_t txn_record_carried_frame_t txn_record_decode_frame_t @@ -7018,12 +7125,15 @@ txn_record_encode_frame_t txn_record_encoding_frame_t txn_record_freed_frame_t txn_record_pieces_frame_t +txn_record_unlisted_frame_t txn_record_written_frame_t +txn_rewrite_Room_t txn_rewrite_based_closure_0_captures_t txn_rewrite_based_closure_0_frame_t txn_rewrite_based_frame_t txn_rewrite_carried_closure_5_captures_t txn_rewrite_carried_closure_5_frame_t +txn_rewrite_carried_frame_1_t txn_rewrite_carried_frame_t txn_rewrite_keys_frame_t txn_rewrite_new_base_frame_t @@ -7054,31 +7164,43 @@ txn_runs_tests_chunk_captures_t txn_runs_tests_cut_captures_t txn_runs_tests_entry_option_t txn_runs_tests_holds_both_t +txn_runs_tests_identity_t txn_runs_tests_index_entry_t txn_runs_tests_kept_t txn_runs_tests_key_count_t txn_runs_tests_keyed_t txn_runs_tests_loaded_captures_t +txn_runs_tests_merge_t txn_runs_tests_row_captures_t txn_runs_tests_trees_t txn_runs_write_frame_0_t txn_runs_written_frame_0_t txn_spill_aborted_captures_t +txn_spill_aborted_pending_t txn_spill_at_found_frame_0_t +txn_spill_carried_frame_0_t txn_spill_changed_frame_0_t txn_spill_chunk_frame_0_t txn_spill_chunks_ended_frame_0_t txn_spill_clone_frame_0_t +txn_spill_committed_frame_0_t +txn_spill_committed_oldest_t +txn_spill_file_counted_frame_0_t txn_spill_files_frame_0_t txn_spill_folded_frame_0_t txn_spill_from_frame_0_t +txn_spill_group_written_frame_0_t txn_spill_held_frame_0_t txn_spill_helped_frame_0_t txn_spill_hits_frame_0_t +txn_spill_identity_applies_frame_0_t +txn_spill_identity_folded_frame_0_t +txn_spill_identity_saved_frame_0_t txn_spill_joined_frame_0_t txn_spill_linked_captures_t txn_spill_listed_frame_0_t txn_spill_looked_up_captures_t +txn_spill_marked_t txn_spill_merged_frame_0_t txn_spill_named_frame_0_t txn_spill_newest_value_captures_t @@ -7087,19 +7209,27 @@ txn_spill_next_frame_0_t txn_spill_pended_frame_0_t txn_spill_read_frame_0_t txn_spill_reading_frame_0_t +txn_spill_redo_aborted_frame_0_t txn_spill_removed_captures_t txn_spill_removed_frame_0_t txn_spill_removed_keys_frame_0_t txn_spill_replayed_captures_t txn_spill_replayed_frame_0_t +txn_spill_residual_captures_t +txn_spill_residual_frame_0_t +txn_spill_rewrote_frame_0_t txn_spill_settle_pending_captures_t txn_spill_settled_from_frame_0_t txn_spill_spilled_turn_captures_t +txn_spill_spillers_frame_0_t +txn_spill_spillers_t txn_spill_swept_frame_0_t txn_spill_tests_based_t +txn_spill_tests_carry_t txn_spill_tests_found_t txn_spill_tests_kind_number_t txn_spill_tests_read_t +txn_spill_tests_residual_t txn_spill_tests_three_t txn_spill_tests_versioned_t txn_spill_tests_writing_t @@ -7107,6 +7237,10 @@ txn_spill_tuple_key_frame_0_t txn_spill_tupled_frame_0_t txn_spill_twinned_captures_t txn_spill_unfolded_frame_0_t +txn_spill_unlisted_frame_t +txn_spill_unlisted_redo_captures_t +txn_spill_unlisted_redo_frame_t +txn_spill_unlisted_run_frame_t txn_spill_unreplayed_captures_t txn_spill_versions_frame_0_t txn_state_Saving_saved_frame_0_t @@ -7143,6 +7277,7 @@ txn_state_state_frame_0_t txn_state_states_frame_0_t txn_state_sweep_frame_0_t txn_state_tests_based_t +txn_state_tests_keying_t txn_state_tests_opened_t txn_state_tests_put_t txn_state_tests_saved_t @@ -7158,6 +7293,7 @@ txn_status_lowered_frame_t txn_status_remove_frame_t txn_status_take_frame_t txn_vacuum_BlockStep_t +txn_vacuum_Read_t txn_vacuum_TupleU64Bool_t txn_vacuum_block_frame_t txn_vacuum_freed_frame_t @@ -7165,6 +7301,7 @@ txn_vacuum_given_closure_2_captures_t txn_vacuum_given_closure_2_frame_t txn_vacuum_given_frame_t txn_vacuum_holding_frame_t +txn_vacuum_leaving_frame_t txn_vacuum_mapped_frame_t txn_vacuum_never_held_frame_t txn_vacuum_pended_frame_t @@ -7177,7 +7314,21 @@ txn_vacuum_redo_freed_closure_5_captures_t txn_vacuum_spent_frame_t txn_vacuum_taken_frame_t txn_vacuum_taken_out_frame_t +txn_vacuum_tests_aborted_t +txn_vacuum_tests_case_t +txn_vacuum_tests_counted_t +txn_vacuum_tests_entry_t +txn_vacuum_tests_finding_t +txn_vacuum_tests_leaving_t +txn_vacuum_tests_logged_t +txn_vacuum_tests_made_t +txn_vacuum_tests_pruning_t +txn_vacuum_tests_raced_t +txn_vacuum_tests_redoing_t +txn_vacuum_tests_skipped_t txn_values_entry_frame_t +txn_values_tests_made_t +txn_values_tests_written_t txn_visible_live_frame_t txn_visible_lockers_frame_t txn_visible_next_frame_t @@ -7185,6 +7336,7 @@ txn_visible_seen_frame_t txn_visible_unfolded_frame_t txn_visible_updatable_frame_t txn_written_Bytes_t +txn_written_Homed_t txn_written_Keys_t txn_written_Largest_t txn_written_Mark_t