From 17709f2e7fa4a8e13ebacd927f7ad20c05faf200 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 15 Jun 2026 05:32:51 +0800 Subject: [PATCH 1/8] refactor(freelist): relocate free-list from catalog B+ tree to durable header chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The persistent free-list was stored as individual catalog rows keyed `[0x04]||page_id_be` with deferred-free pairs under key `[0x05]`. Every commit that freed pages had to copy-on-write the catalog tree to update these rows, adding per-commit overhead proportional to the number of freed pages and causing unbounded catalog churn under sustained writes. Replace this with a chain of `PageKind::Free` pages rooted in the A/B header's `free_list_root` slot. Each chain page stores `(commit_id, page_id)` pairs in a compact binary layout; the chain is rewritten atomically with the header swap, so it survives unclean shutdown without catalog involvement. Benefits: - Freed pages drain to the allocator cache on commit without touching the catalog tree, eliminating per-commit catalog CoW overhead. - The reclamation floor (oldest reader pin or oldest retained history commit, whichever is lower) is computed once at `begin_write`; pages below it are safe to recycle immediately within the same txn. - Compaction no longer needs to partition catalog rows into free-list vs non-free-list groups: the dense repack simply resets `free_list_root` to 0 and the catalog stays clean. Remove `compaction::freelist` (the catalog-based helpers), strip the `FreeList` / `DeferredFree` catalog row kinds and their codec, and drop the `skip_freelist_persistence_when_no_readers` open option (the new path always recycles eligible pages without a catalog write, making the flag's trade-off moot). Add `pager::freelist` with `read_chain` and `rewrite_chain` — the two primitives used by the commit and compaction paths. --- src/catalog/codec.rs | 63 +------------- src/compaction/freelist.rs | 129 ----------------------------- src/compaction/full.rs | 163 +++++------------------------------- src/compaction/helpers.rs | 55 +++++-------- src/compaction/mod.rs | 11 ++- src/compaction/step.rs | 135 ++++++------------------------ src/options.rs | 39 --------- src/pager/freelist.rs | 164 +++++++++++++++++++++++++++++++++++++ src/pager/mod.rs | 1 + 9 files changed, 240 insertions(+), 520 deletions(-) delete mode 100644 src/compaction/freelist.rs create mode 100644 src/pager/freelist.rs diff --git a/src/catalog/codec.rs b/src/catalog/codec.rs index 0bf0ca3..12e283b 100644 --- a/src/catalog/codec.rs +++ b/src/catalog/codec.rs @@ -23,16 +23,9 @@ pub enum CatalogRowKind { /// (singleton; no name suffix). Value is `RekeyState` encoded as 13 bytes: /// `target_mk_epoch[8] || main_db_done[1] || segments_remaining_idx[4]`. RekeyState = 0x03, - /// Persistent free-list entry. Key is `[0x04] || page_id_big_endian[8]` - /// (big-endian so lexicographic sort matches numeric order). Value is empty. - FreeList = 0x04, - /// Deferred-free queue. Singleton row; key is `[0x05]`. Value is a - /// length-prefixed list of `(commit_id[8] || page_id[8])` pairs, in - /// ascending `commit_id` order. Pages here are not yet safe to reallocate - /// because a reader snapshot at or before that `commit_id` may still observe - /// them. Pairs are promoted to the free-list once no tracked reader pins - /// a `commit_id` ≤ the pair's `commit_id`. - DeferredFree = 0x05, + // 0x04 and 0x05 are reserved: they were the in-catalog free-list and + // deferred-free queue, superseded by the durable free-list chain rooted in + // the A/B header (see `crate::pager::freelist`). Do not reuse these bytes. /// Durable reader pin. One row per active cross-process read transaction. /// Key: `[0x06] || pid_u32_be[4] || lease_id_u64_be[8]` (13 bytes). /// Value: `commit_id[8] || root_page_id[8] || catalog_root_page_id[8] || @@ -200,56 +193,6 @@ impl Catalog { vec![CatalogRowKind::RekeyState as u8] } - /// Free-list entry key: `[0x04] || page_id_be[8]`. - /// Big-endian so lexicographic order == numeric order. - #[must_use] - pub fn free_list_key(page_id: u64) -> [u8; 9] { - let mut k = [0u8; 9]; - k[0] = CatalogRowKind::FreeList as u8; - k[1..9].copy_from_slice(&page_id.to_be_bytes()); - k - } - - /// Deferred-free queue key: `[0x05]` (singleton). - #[must_use] - pub fn deferred_free_key() -> Vec { - vec![CatalogRowKind::DeferredFree as u8] - } - - /// Encode a slice of `(commit_id, page_id)` pairs. Empty slice encodes to - /// empty bytes. - #[must_use] - pub fn encode_deferred_free(pairs: &[(u64, u64)]) -> Vec { - let mut out = Vec::with_capacity(pairs.len() * 16); - for (commit_id, page_id) in pairs { - out.extend_from_slice(&commit_id.to_le_bytes()); - out.extend_from_slice(&page_id.to_le_bytes()); - } - out - } - - /// Decode a deferred-free value. Returns `Err` if length is not a multiple - /// of 16. - pub fn decode_deferred_free(bytes: &[u8]) -> Result> { - if bytes.len() % 16 != 0 { - return Err(PagedbError::corruption( - crate::errors::CorruptionDetail::HeaderUnverifiable, - )); - } - let mut out = Vec::with_capacity(bytes.len() / 16); - let mut i = 0; - while i + 16 <= bytes.len() { - let mut b8 = [0u8; 8]; - b8.copy_from_slice(&bytes[i..i + 8]); - let commit_id = u64::from_le_bytes(b8); - b8.copy_from_slice(&bytes[i + 8..i + 16]); - let page_id = u64::from_le_bytes(b8); - out.push((commit_id, page_id)); - i += 16; - } - Ok(out) - } - /// Encode a `RekeyStateRow` as 13 bytes. #[must_use] pub fn encode_rekey_state(r: &RekeyStateRow) -> [u8; REKEY_STATE_LEN] { diff --git a/src/compaction/freelist.rs b/src/compaction/freelist.rs deleted file mode 100644 index c182c36..0000000 --- a/src/compaction/freelist.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! Persistent free-list operations. -//! -//! The free-list is stored in the catalog B+ tree as individual rows keyed -//! `[0x04] || page_id_be[8]`. The deferred-free queue is a single catalog row -//! `[0x05]` containing serialised `(commit_id, page_id)` pairs. -//! -//! These helpers are used both by [`WriteTxn::commit`] (to drain freed pages -//! into the deferred queue) and by [`compact_now`] (to promote eligible -//! deferred pages into the free-list and to allocate from it). - -use std::sync::Arc; - -use crate::btree::BTree; -use crate::catalog::codec::{Catalog, CatalogRowKind}; -use crate::pager::Pager; -use crate::vfs::Vfs; -use crate::{RealmId, Result}; - -/// Pop the smallest `page_id` from the persistent free-list. Returns `None` if -/// the free-list is empty. -/// -/// Modifies `cat_tree` in-place (deletes the chosen entry). The caller is -/// responsible for flushing `cat_tree` and committing the header afterward. -pub async fn alloc_page(cat_tree: &mut BTree) -> Result> { - let start = vec![CatalogRowKind::FreeList as u8]; - let mut end = start.clone(); - end.push(0xFF); - let rows = cat_tree.collect_range(&start, &end).await?; - if let Some((key, _)) = rows.into_iter().next() { - // Key layout: [0x04] || page_id_be[8] — 9 bytes total. - if key.len() == 9 { - let mut b = [0u8; 8]; - b.copy_from_slice(&key[1..9]); - let page_id = u64::from_be_bytes(b); - cat_tree.delete(&key).await?; - return Ok(Some(page_id)); - } - } - Ok(None) -} - -/// Add `page_id` to the deferred-free queue tagged with `commit_id`. The page -/// becomes eligible for reuse once no tracked reader pins a `commit_id` ≤ this -/// `commit_id`. -/// -/// Modifies `cat_tree` in-place. Caller flushes and commits. -pub async fn free_page_deferred( - cat_tree: &mut BTree, - commit_id: u64, - page_id: u64, -) -> Result { - free_pages_deferred_batch(cat_tree, commit_id, &[page_id]).await -} - -/// Batch-add multiple `page_id`s to the deferred-free queue in a single -/// catalog put. More efficient than calling `free_page_deferred` in a loop -/// when many pages need to be freed at once. -/// -/// Modifies `cat_tree` in-place. Caller flushes and commits. Returns the -/// total pair count in the deferred-free row after the append; callers -/// (e.g. the stall-policy check) can use this to avoid re-reading the row. -pub async fn free_pages_deferred_batch( - cat_tree: &mut BTree, - commit_id: u64, - page_ids: &[u64], -) -> Result { - if page_ids.is_empty() { - return Ok(0); - } - let dk = Catalog::deferred_free_key(); - let mut pairs = match cat_tree.get(&dk).await? { - Some(bytes) => Catalog::decode_deferred_free(&bytes)?, - None => Vec::new(), - }; - for &page_id in page_ids { - pairs.push((commit_id, page_id)); - } - // Keep sorted by commit_id for efficient draining. - pairs.sort_unstable_by_key(|(cid, _)| *cid); - let encoded = Catalog::encode_deferred_free(&pairs); - cat_tree.put(&dk, &encoded).await?; - Ok(pairs.len() as u64) -} - -/// Drain all deferred-free entries with `commit_id < min_reader_commit` into -/// the persistent free-list. Returns the number of pages promoted. -/// -/// Modifies `cat_tree` in-place. Caller flushes and commits. -pub async fn drain_deferred_to_freelist( - cat_tree: &mut BTree, - min_reader_commit: u64, -) -> Result { - let dk = Catalog::deferred_free_key(); - let pairs = match cat_tree.get(&dk).await? { - Some(bytes) => Catalog::decode_deferred_free(&bytes)?, - None => return Ok(0), - }; - - let (eligible, remaining): (Vec<_>, Vec<_>) = pairs - .into_iter() - .partition(|(cid, _)| *cid < min_reader_commit); - - let count = eligible.len() as u64; - for (_cid, page_id) in eligible { - let key = Catalog::free_list_key(page_id); - cat_tree.put(&key, &[]).await?; - } - - if remaining.is_empty() { - let _ = cat_tree.delete(&dk).await?; - } else { - let encoded = Catalog::encode_deferred_free(&remaining); - cat_tree.put(&dk, &encoded).await?; - } - - Ok(count) -} - -/// Persist the updated catalog state after free-list operations. This is a -/// low-level helper; callers that already hold the writer lock and have -/// modified a `BTree` should flush+commit themselves. Provided here for -/// completeness. -pub async fn persist_freelist_state( - _pager: &Arc>, - _realm_id: RealmId, -) -> Result<()> { - // Persistence is handled by the caller through the normal commit path. - Ok(()) -} diff --git a/src/compaction/full.rs b/src/compaction/full.rs index efc08e5..1c5533e 100644 --- a/src/compaction/full.rs +++ b/src/compaction/full.rs @@ -9,7 +9,6 @@ //! 6. Repacks segment files whose garbage ratio exceeds 5%. use crate::btree::BTree; -use crate::catalog::codec::Catalog; use crate::errors::PagedbError; use crate::pager::header::commit_header; use crate::pager::structural_header::MainDbHeaderFields; @@ -39,14 +38,20 @@ pub async fn compact_now(db: &Db) -> Result { // Acquire exclusive writer lock for the entire compact operation. let mut state = db.writer.lock().await; + // Compaction relocates and/or truncates pages, invalidating every page id + // cached for runtime reuse. Drop those reuse hints so a post-compaction + // commit can't recycle a page the repack now uses for live data. + db.free_page_cache.lock().clear(); + let old_next_page_id = state.next_page_id; - // ── 1. Check reader pins ────────────────────────────────────────────────── - // If any reader is pinned, the main tree pages are still live in their - // snapshot. We must not overwrite them with the bulk-load repack. Instead, - // only drain eligible deferred-free entries via CoW (which appends pages - // at the high end, above the reader range) and skip truncation. - let (has_readers, min_reader_commit) = { + // ── 1. Refuse while readers are pinned ─────────────────────────────────── + // A dense repack relocates the current tree and truncates the file; pinned + // readers (in-process or cross-process durable) still reference the old + // pages, so neither is safe under them. Runtime free-page reuse already + // reclaims space on ordinary commits, so there is nothing for compaction to + // do here until the readers drop. + let has_readers = { let in_mem_min = { let readers = db.tracked_readers.lock(); readers @@ -55,117 +60,12 @@ pub async fn compact_now(db: &Db) -> Result { .min() .unwrap_or(u64::MAX) }; - // Also check cross-process durable reader pins so that foreign readers - // whose pins are not in-memory are not accidentally reclaimed. let durable_min = db .min_durable_reader_commit(state.catalog_root_page_id, state.next_page_id) .await; - let min = in_mem_min.min(durable_min); - let has = min < u64::MAX; - (has, min) + in_mem_min.min(durable_min) < u64::MAX }; - if has_readers { - // Limited compaction: only drain eligible deferred-free entries into - // the persistent free-list using CoW (safe — appends above existing range). - let (cat_non_housekeeping, deferred_pairs) = - collect_catalog_split(&db.pager, db.realm_id, &state).await?; - - let (eligible_free, still_deferred): (Vec<_>, Vec<_>) = deferred_pairs - .into_iter() - .partition(|(commit_id, _)| *commit_id < min_reader_commit); - - if eligible_free.is_empty() { - // Nothing to do while readers are pinned. - return Ok(result); - } - - // Drain eligible deferred pages into the free-list using CoW. - let mut cat_tree = BTree::open( - db.pager.clone(), - db.realm_id, - state.catalog_root_page_id, - state.next_page_id, - db.page_size, - ); - - // Write free-list rows for eligible pages. - for (_cid, page_id) in &eligible_free { - let key = Catalog::free_list_key(*page_id); - cat_tree.put(&key, &[]).await?; - } - // Update the deferred-free row. - let dk = Catalog::deferred_free_key(); - if still_deferred.is_empty() { - let _ = cat_tree.delete(&dk).await; - } else { - let encoded = Catalog::encode_deferred_free(&still_deferred); - cat_tree.put(&dk, &encoded).await?; - } - - // Merge any new catalog content collected (non-housekeeping stays). - let _ = cat_non_housekeeping; - - cat_tree.flush().await?; - let new_cat_root = cat_tree.root_page_id(); - let new_next = cat_tree.next_page_id().max(state.next_page_id); - - let new_commit_id = state.latest_commit_id + 1; - let new_seq = state.seq + 1; - let counter_anchor = db.pager.pending_anchor(); - - let mut catalog_root_bytes = [0u8; 16]; - catalog_root_bytes[..8].copy_from_slice(&new_cat_root.to_le_bytes()); - catalog_root_bytes[8..].copy_from_slice(&new_commit_id.to_le_bytes()); - - let fields = MainDbHeaderFields { - format_version: 1, - cipher_id: db.cipher_id.as_byte(), - page_size_log2: page_size_log2(db.page_size)?, - flags: 0, - file_id: db.file_id, - kek_salt: db.kek_salt, - mk_epoch: db.mk_epoch.load(std::sync::atomic::Ordering::SeqCst), - seq: new_seq, - active_root_page_id: state.root_page_id, - active_root_txn_id: state.latest_commit_id, - counter_anchor, - commit_id: CommitId(new_commit_id), - free_list_root: [0u8; 16], - catalog_root: catalog_root_bytes, - apply_journal_root_page_id: 0, - apply_journal_root_version: 0, - commit_history_root_page_id: 0, - commit_history_root_version: 0, - restore_mode: 0, - next_page_id: new_next, - commit_retain_policy_tag: 0, - commit_retain_policy_value: 0, - }; - - let hk_clone = { db.hk.read().clone() }; - let new_slot = commit_header( - &*db.vfs, - &db.main_db_path, - &hk_clone, - &fields, - state.active_slot, - db.page_size, - ) - .await?; - db.pager.commit_anchor(counter_anchor)?; - - state.catalog_root_page_id = new_cat_root; - state.next_page_id = new_next; - state.active_slot = new_slot; - state.seq = new_seq; - state.latest_commit_id = new_commit_id; - state.commit_history_root_page_id = 0; - state.commit_history_root_version = 0; - db.latest_commit - .store(new_commit_id, std::sync::atomic::Ordering::SeqCst); - - result.main_db_pages_reclaimed = eligible_free.len() as u64; return Ok(result); } @@ -185,16 +85,8 @@ pub async fn compact_now(db: &Db) -> Result { Vec::new() }; - // Collect catalog pairs, separating out free-list and deferred-free rows - // so we can rebuild them correctly in the new tree. - let (cat_non_housekeeping, deferred_pairs) = - collect_catalog_split(&db.pager, db.realm_id, &state).await?; - - let (eligible_free, still_deferred): (Vec<_>, Vec<_>) = deferred_pairs - .into_iter() - .partition(|(commit_id, _)| *commit_id < min_reader_commit); - - let free_pages_before = eligible_free.len() as u64; + // Collect catalog rows (housekeeping free-list rows dropped). + let cat_rows = collect_catalog_split(&db.pager, db.realm_id, &state).await?; // ── 3. Write fresh compacted trees starting at page 4 ──────────────────── // Pages 0–3 are reserved (header slots A/B + two spares); never allocated. @@ -210,29 +102,15 @@ pub async fn compact_now(db: &Db) -> Result { let new_root = new_main.root_page_id(); let after_main = new_main.next_page_id(); - // Build new catalog tree via bulk-load. Include all non-housekeeping rows, - // plus the remaining deferred-free (if any). - // NOTE: eligible_free pages are skipped: after truncation, pages above - // new_next don't exist; pages below new_next are fully occupied by the - // compacted layout. Free-list management during normal writes is handled - // in WriteTxn::commit via free_page_deferred. - let mut cat_all: Vec<(Vec, Vec)> = cat_non_housekeeping; - if !still_deferred.is_empty() { - let dk = Catalog::deferred_free_key(); - let encoded = Catalog::encode_deferred_free(&still_deferred); - cat_all.push((dk, encoded)); - } - cat_all.sort_by(|(a, _), (b, _)| a.cmp(b)); let mut new_cat = BTree::open(db.pager.clone(), db.realm_id, 0, after_main, db.page_size); - new_cat.bulk_load(cat_all).await?; + new_cat.bulk_load(cat_rows).await?; new_cat.flush().await?; let new_cat_root = new_cat.root_page_id(); let new_next = new_cat.next_page_id(); - // Pages reclaimed = reduction in next_page_id + eligible free pages consumed. - result.main_db_pages_reclaimed = old_next_page_id - .saturating_sub(new_next) - .saturating_add(free_pages_before); + // Pages reclaimed = reduction in next_page_id (the dense layout is contiguous, + // and the durable free-list is reset to empty below). + result.main_db_pages_reclaimed = old_next_page_id.saturating_sub(new_next); // ── 4. Commit new header ───────────────────────────────────────────────── let new_commit_id = state.latest_commit_id + 1; @@ -288,6 +166,9 @@ pub async fn compact_now(db: &Db) -> Result { state.latest_commit_id = new_commit_id; state.commit_history_root_page_id = 0; state.commit_history_root_version = 0; + // The dense repack relocates/truncates every page, so the old free-list is + // gone; the new layout starts with an empty free-list. + state.free_list_root_page_id = 0; db.latest_commit .store(new_commit_id, std::sync::atomic::Ordering::SeqCst); diff --git a/src/compaction/helpers.rs b/src/compaction/helpers.rs index 1441af1..aaf5c7d 100644 --- a/src/compaction/helpers.rs +++ b/src/compaction/helpers.rs @@ -21,19 +21,17 @@ pub(super) async fn collect_all_pairs( tree.collect_range(&[], &[0xFF; 256]).await } -/// Collect catalog pairs split into two groups: -/// - `non_housekeeping`: all rows that are NOT free-list or deferred-free rows. -/// - `deferred_pairs`: the decoded deferred-free queue entries. -/// -/// Free-list rows are intentionally excluded; they will be reconstructed -/// during compaction by promoting eligible deferred-free entries. +/// Collect every catalog row, for rebuilding the catalog tree during a dense +/// repack. Free pages are tracked in the durable free-list chain (reset +/// separately by the repack), not in the catalog, so there are no housekeeping +/// rows to filter here. pub(super) async fn collect_catalog_split( pager: &Arc>, realm_id: crate::RealmId, state: &WriterState, -) -> Result<(Vec<(Vec, Vec)>, Vec<(u64, u64)>)> { +) -> Result, Vec)>> { if state.catalog_root_page_id == 0 { - return Ok((Vec::new(), Vec::new())); + return Ok(Vec::new()); } let tree = BTree::open( pager.clone(), @@ -42,28 +40,7 @@ pub(super) async fn collect_catalog_split( state.next_page_id, pager.page_size(), ); - let all = collect_all_pairs(&tree).await?; - - let fl_byte = CatalogRowKind::FreeList as u8; - let df_key = Catalog::deferred_free_key(); - - let mut non_housekeeping: Vec<(Vec, Vec)> = Vec::new(); - let mut deferred_pairs: Vec<(u64, u64)> = Vec::new(); - - for (k, v) in all { - if k.first() == Some(&fl_byte) { - // Free-list row — drop; will be rebuilt from eligible deferred entries. - continue; - } - if k == df_key { - // Deferred-free row — decode separately. - deferred_pairs = Catalog::decode_deferred_free(&v)?; - continue; - } - non_housekeeping.push((k, v)); - } - - Ok((non_housekeeping, deferred_pairs)) + collect_all_pairs(&tree).await } pub(super) async fn list_all_segments_inner( @@ -284,7 +261,15 @@ pub(super) async fn load_frontier_page_id( } } -/// Build [`MainDbHeaderFields`] for a commit. +/// Build [`MainDbHeaderFields`] for a compaction commit. +/// +/// Compaction relocates/truncates pages, so it discards the commit-history +/// index (`commit_history_root = 0`) — exactly as `compact_now` does. Writing +/// the *old* history root here would leave the durable header pointing at a +/// history tree the repack just overwrote/truncated, so a later `begin_read_at` +/// would read garbage. `free_list_root` is supplied by the caller: an +/// intermediate step preserves the still-valid chain; the final dense repack +/// passes 0 (the relocated layout starts with an empty free-list). #[allow(clippy::too_many_arguments)] pub(super) fn make_header_fields( db: &Db, @@ -295,7 +280,9 @@ pub(super) fn make_header_fields( new_root: u64, new_cat_root: u64, new_next: u64, + free_list_root_page_id: u64, ) -> MainDbHeaderFields { + let _ = state; let mut catalog_root_bytes = [0u8; 16]; catalog_root_bytes[..8].copy_from_slice(&new_cat_root.to_le_bytes()); catalog_root_bytes[8..].copy_from_slice(&new_commit_id.to_le_bytes()); @@ -312,12 +299,12 @@ pub(super) fn make_header_fields( active_root_txn_id: new_commit_id, counter_anchor, commit_id: CommitId(new_commit_id), - free_list_root: [0u8; 16], + free_list_root: crate::txn::db::encode_free_list_root(free_list_root_page_id), catalog_root: catalog_root_bytes, apply_journal_root_page_id: 0, apply_journal_root_version: 0, - commit_history_root_page_id: state.commit_history_root_page_id, - commit_history_root_version: state.commit_history_root_version, + commit_history_root_page_id: 0, + commit_history_root_version: 0, restore_mode: 0, next_page_id: new_next, commit_retain_policy_tag: 0, diff --git a/src/compaction/mod.rs b/src/compaction/mod.rs index ecc584a..3bfc431 100644 --- a/src/compaction/mod.rs +++ b/src/compaction/mod.rs @@ -1,18 +1,17 @@ -//! Online compaction: persistent free-list management, main.db defragmentation, -//! and segment repacking. +//! Online compaction: main.db defragmentation and segment repacking. +//! +//! Free-page reclamation itself is handled at runtime by the durable free-list +//! ([`crate::pager::freelist`]); compaction's job is the dense repack + file +//! truncation and segment garbage collection. //! //! Entry points: [`compact_now`] (full one-shot) and [`compact_step`] //! (incremental, budget-bounded) on a [`crate::Db`] handle. -pub mod freelist; mod full; mod helpers; mod step; mod types; -pub use freelist::{ - alloc_page, drain_deferred_to_freelist, free_page_deferred, persist_freelist_state, -}; pub use full::compact_now; pub use step::compact_step; pub use types::{CompactBudget, CompactProgress, CompactStats}; diff --git a/src/compaction/step.rs b/src/compaction/step.rs index 8dbb78a..780fef6 100644 --- a/src/compaction/step.rs +++ b/src/compaction/step.rs @@ -8,7 +8,6 @@ use crate::pager::header::commit_header; use crate::txn::db::Db; use crate::vfs::{Vfs, VfsFile}; -use super::freelist::drain_deferred_to_freelist; use super::helpers::{ collect_all_pairs, collect_catalog_split, collect_range_limited, load_compaction_state, load_frontier_page_id, make_header_fields, next_key_after, @@ -35,8 +34,14 @@ pub async fn compact_step( let mut state = db.writer.lock().await; - // ── Determine reader presence ───────────────────────────────────────────── - let (has_readers, min_reader_commit) = { + // Relocation/truncation below invalidates cached reuse page ids; drop them. + db.free_page_cache.lock().clear(); + + // ── Refuse while readers are pinned ─────────────────────────────────────── + // Relocation/truncation is unsafe under a pinned reader (in-process or + // cross-process durable). Runtime free-page reuse reclaims space on ordinary + // commits, so compaction simply waits for the readers to drop. + let has_readers = { let in_mem_min = { let readers = db.tracked_readers.lock(); readers @@ -48,89 +53,12 @@ pub async fn compact_step( let durable_min = db .min_durable_reader_commit(state.catalog_root_page_id, state.next_page_id) .await; - let min = in_mem_min.min(durable_min); - let has = min < u64::MAX; - (has, min) + in_mem_min.min(durable_min) < u64::MAX }; - if has_readers { - // With readers pinned we can only drain eligible deferred-free entries. - let (cat_non_housekeeping, deferred_pairs) = - collect_catalog_split(&db.pager, db.realm_id, &state).await?; - let (eligible_free, still_deferred): (Vec<_>, Vec<_>) = deferred_pairs - .into_iter() - .partition(|(commit_id, _)| *commit_id < min_reader_commit); - - if eligible_free.is_empty() { - let wm = load_frontier_page_id(&db.pager, db.realm_id, &state).await; - return Ok(CompactProgress { - pages_relocated: 0, - bytes_freed: 0, - more_work: true, - watermark: wm, - }); - } - - let mut cat_tree = BTree::open( - db.pager.clone(), - db.realm_id, - state.catalog_root_page_id, - state.next_page_id, - db.page_size, - ); - for (_cid, page_id) in &eligible_free { - let key = Catalog::free_list_key(*page_id); - cat_tree.put(&key, &[]).await?; - } - let dk = Catalog::deferred_free_key(); - if still_deferred.is_empty() { - let _ = cat_tree.delete(&dk).await; - } else { - let encoded = Catalog::encode_deferred_free(&still_deferred); - cat_tree.put(&dk, &encoded).await?; - } - let _ = cat_non_housekeeping; - cat_tree.flush().await?; - let new_cat_root = cat_tree.root_page_id(); - let new_next = cat_tree.next_page_id().max(state.next_page_id); - - let new_commit_id = state.latest_commit_id + 1; - let new_seq = state.seq + 1; - let counter_anchor = db.pager.pending_anchor(); - let fields = make_header_fields( - db, - &state, - new_commit_id, - new_seq, - counter_anchor, - state.root_page_id, - new_cat_root, - new_next, - ); - let hk_clone = { db.hk.read().clone() }; - let new_slot = commit_header( - &*db.vfs, - &db.main_db_path, - &hk_clone, - &fields, - state.active_slot, - db.page_size, - ) - .await?; - db.pager.commit_anchor(counter_anchor)?; - state.catalog_root_page_id = new_cat_root; - state.next_page_id = new_next; - state.active_slot = new_slot; - state.seq = new_seq; - state.latest_commit_id = new_commit_id; - state.commit_history_root_page_id = 0; - state.commit_history_root_version = 0; - db.latest_commit - .store(new_commit_id, std::sync::atomic::Ordering::SeqCst); - let wm = load_frontier_page_id(&db.pager, db.realm_id, &state).await; return Ok(CompactProgress { - pages_relocated: eligible_free.len() as u64, + pages_relocated: 0, bytes_freed: 0, more_work: true, watermark: wm, @@ -202,14 +130,7 @@ pub async fn compact_step( Vec::new() }; - let (cat_non_housekeeping, deferred_pairs) = - collect_catalog_split(&db.pager, db.realm_id, &state).await?; - - let (eligible_free, still_deferred): (Vec<_>, Vec<_>) = deferred_pairs - .into_iter() - .partition(|(commit_id, _)| *commit_id < min_reader_commit); - - let free_pages_before = eligible_free.len() as u64; + let cat_rows = collect_catalog_split(&db.pager, db.realm_id, &state).await?; let mut new_main = BTree::open(db.pager.clone(), db.realm_id, 0, 4, db.page_size); new_main.bulk_load(main_pairs).await?; @@ -219,15 +140,10 @@ pub async fn compact_step( // Remove compaction-state row from the catalog being rebuilt. let cs_key_prefix = crate::catalog::codec::CatalogRowKind::CompactionState as u8; - let mut cat_all: Vec<(Vec, Vec)> = cat_non_housekeeping + let mut cat_all: Vec<(Vec, Vec)> = cat_rows .into_iter() .filter(|(k, _)| k.first() != Some(&cs_key_prefix)) .collect(); - if !still_deferred.is_empty() { - let dk = Catalog::deferred_free_key(); - let encoded = Catalog::encode_deferred_free(&still_deferred); - cat_all.push((dk, encoded)); - } cat_all.sort_by(|(a, _), (b, _)| a.cmp(b)); let mut new_cat = BTree::open(db.pager.clone(), db.realm_id, 0, after_main, db.page_size); new_cat.bulk_load(cat_all).await?; @@ -235,13 +151,12 @@ pub async fn compact_step( let new_cat_root = new_cat.root_page_id(); let new_next = new_cat.next_page_id(); - let pages_reclaimed = old_next_page_id - .saturating_sub(new_next) - .saturating_add(free_pages_before); + let pages_reclaimed = old_next_page_id.saturating_sub(new_next); let new_commit_id = state.latest_commit_id + 1; let new_seq = state.seq + 1; let counter_anchor = db.pager.pending_anchor(); + // Dense repack: the relocated layout starts with an empty free-list. let fields = make_header_fields( db, &state, @@ -251,6 +166,7 @@ pub async fn compact_step( new_root, new_cat_root, new_next, + 0, ); let hk_clone = { db.hk.read().clone() }; let new_slot = commit_header( @@ -272,6 +188,7 @@ pub async fn compact_step( state.latest_commit_id = new_commit_id; state.commit_history_root_page_id = 0; state.commit_history_root_version = 0; + state.free_list_root_page_id = 0; db.latest_commit .store(new_commit_id, std::sync::atomic::Ordering::SeqCst); @@ -301,7 +218,6 @@ pub async fn compact_step( .last() .map_or_else(|| frontier_key.clone(), |(k, _)| k.clone()); - // Build catalog tree with deferred-free drained, then update watermark. let mut cat_tree = BTree::open( db.pager.clone(), db.realm_id, @@ -309,12 +225,11 @@ pub async fn compact_step( state.next_page_id, db.page_size, ); - // Drain all deferred-free into free-list (no readers pinned). - drain_deferred_to_freelist(&mut cat_tree, u64::MAX).await?; // Re-insert the batch: delete+put forces page reallocation to low-address - // free slots. Pages freed by delete are added to the free-list immediately - // (no readers), so the subsequent put can reuse them. + // free slots. Pages freed by the delete are recycled within this same txn + // (the allocator pops the per-session freed list), so the subsequent put + // reuses them — densifying the layout into the low page-id range. let mut main_tree = BTree::open( db.pager.clone(), db.realm_id, @@ -327,13 +242,6 @@ pub async fn compact_step( main_tree.put(k, v).await?; } - // Freed pages from main_tree deletions: add to catalog free-list directly. - let freed_by_tree = main_tree.drain_freed(); - for pid in freed_by_tree { - let key = Catalog::free_list_key(pid); - cat_tree.put(&key, &[]).await?; - } - main_tree.flush().await?; // main_tree.flush() may allocate pages while materializing the dirty-leaf // cache. Sync the catalog tree forward so its flush doesn't reuse a @@ -371,6 +279,10 @@ pub async fn compact_step( let new_commit_id = state.latest_commit_id + 1; let new_seq = state.seq + 1; let counter_anchor = db.pager.pending_anchor(); + // An intermediate step only relocates main-tree pages; it touches neither + // the durable free-list chain nor its free pages, so the free-list stays + // valid and is preserved across the step (the pre-existing free pages remain + // reusable by ordinary writes). The final dense-repack batch resets it. let fields = make_header_fields( db, &state, @@ -380,6 +292,7 @@ pub async fn compact_step( new_main_root, final_cat_root, final_next, + state.free_list_root_page_id, ); let hk_clone = { db.hk.read().clone() }; let new_slot = commit_header( diff --git a/src/options.rs b/src/options.rs index 724e3c0..46bece6 100644 --- a/src/options.rs +++ b/src/options.rs @@ -105,32 +105,6 @@ pub struct OpenOptions { /// this many newly-encrypted pages — exceed it and the txn aborts. Large /// bulk loads need a larger budget. Default: 1024. pub anchor_budget: u64, - - /// **Pagedb extension beyond the architecture spec.** When `true` and no - /// readers are pinned at commit time, skip persisting freed pages into - /// the deferred-free queue / persistent free-list. The pages become - /// **orphan pages** in `main.db` — physically allocated but unreferenced - /// — and can only be reclaimed by [`Db::compact`]. - /// - /// Trade-off: - /// - **Pro**: Eliminates one catalog-tree `CoW` per commit. Big - /// single-put write-latency win (~75µs on a 4 KiB page DB). - /// - **Con**: `main.db` grows monotonically during no-reader phases. - /// Embedders MUST schedule periodic [`Db::compact`] to reclaim space, - /// or accept unbounded growth. - /// - /// Default: `false` (spec-conformant; every freed page is tracked). - /// - /// Use cases that should enable this: - /// - Benchmarks measuring against engines without an equivalent free-list - /// - Ephemeral / cache-like deployments that periodically compact or - /// rebuild - /// - Workloads where the embedder can prove the file is short-lived - /// - /// Use cases that should NOT enable this: - /// - Long-running write-heavy stores without compaction infrastructure - /// - Anything where unbounded `main.db` growth is a problem - pub skip_freelist_persistence_when_no_readers: bool, } impl Default for OpenOptions { @@ -145,9 +119,6 @@ impl Default for OpenOptions { observer_retry_count: 3, metrics_enabled: true, anchor_budget: crate::crypto::nonce::DEFAULT_ANCHOR_BUDGET, - // Default: spec-conformant. Embedders that want the fast path - // explicitly opt in. - skip_freelist_persistence_when_no_readers: false, } } } @@ -218,14 +189,4 @@ impl OpenOptions { self.anchor_budget = v; self } - - /// Opt into the no-readers fast-free path. See - /// [`Self::skip_freelist_persistence_when_no_readers`] for the trade-off. - /// Embedders that enable this **must** schedule periodic - /// [`Db::compact`](crate::Db::compact) to reclaim orphan pages. - #[must_use] - pub fn with_skip_freelist_persistence_when_no_readers(mut self, v: bool) -> Self { - self.skip_freelist_persistence_when_no_readers = v; - self - } } diff --git a/src/pager/freelist.rs b/src/pager/freelist.rs new file mode 100644 index 0000000..a6e6e1c --- /dev/null +++ b/src/pager/freelist.rs @@ -0,0 +1,164 @@ +//! Durable free-page list, stored as a chain of `PageKind::Free` pages rooted +//! at the A/B header's `free_list_root` slot — outside the catalog B+ tree. +//! +//! Keeping the free list out of the catalog is what makes free-page recycling +//! both **durable** (it survives an unclean shutdown — the chain is committed +//! atomically with the header swap) and **bounded** (maintaining it never +//! copies-on-writes the catalog tree, so it adds no per-commit catalog churn). +//! +//! Each entry is a `(commit_id, page_id)` pair: `commit_id` is the commit that +//! freed the page, used at `begin_write` to decide which pages are below the +//! reclamation floor (observable by no reader and no retained-history root) and +//! therefore safe to recycle now. The chain stores *every* free page — those +//! still pinned are simply carried forward until the floor advances past them. +//! +//! Page body layout (`PageKind::Free`): +//! ```text +//! [0..8) next chain page id (LE u64; 0 = end of chain) +//! [8..12) entry count in this page (LE u32) +//! [12..) `count` × (commit_id LE u64 ‖ page_id LE u64) +//! ``` + +use std::collections::HashSet; + +use crate::pager::Pager; +use crate::pager::format::data_page::body_capacity; +use crate::pager::format::page_kind::PageKind; +use crate::vfs::Vfs; +use crate::{PagedbError, RealmId, Result}; + +const ENTRY_LEN: usize = 16; +const PAGE_HEADER_LEN: usize = 12; + +/// Number of `(commit_id, page_id)` entries one free-list page can hold. +#[must_use] +pub const fn chain_capacity(page_size: usize) -> usize { + (body_capacity(page_size) - PAGE_HEADER_LEN) / ENTRY_LEN +} + +/// Walk the free-list chain from `head`, returning all `(commit_id, page_id)` +/// entries and the list of page ids the chain itself occupies. `head == 0` is +/// an empty chain. +pub async fn read_chain( + pager: &Pager, + realm_id: RealmId, + head: u64, +) -> Result<(Vec<(u64, u64)>, Vec)> { + let mut entries = Vec::new(); + let mut chain_pages = Vec::new(); + let mut page = head; + while page != 0 { + let guard = pager.read_main_page(page, realm_id, PageKind::Free).await?; + let body = guard.body_ref(); + let mut next_b = [0u8; 8]; + next_b.copy_from_slice(&body[0..8]); + let next = u64::from_le_bytes(next_b); + let mut cnt_b = [0u8; 4]; + cnt_b.copy_from_slice(&body[8..12]); + let count = u32::from_le_bytes(cnt_b) as usize; + for i in 0..count { + let off = PAGE_HEADER_LEN + i * ENTRY_LEN; + let mut cid_b = [0u8; 8]; + cid_b.copy_from_slice(&body[off..off + 8]); + let mut pid_b = [0u8; 8]; + pid_b.copy_from_slice(&body[off + 8..off + 16]); + entries.push((u64::from_le_bytes(cid_b), u64::from_le_bytes(pid_b))); + } + chain_pages.push(page); + page = next; + } + Ok((entries, chain_pages)) +} + +/// Persist `entries` as a fresh chain, returning the new head page id and the +/// updated `next_page` cursor. +/// +/// Chain pages are drawn first from `host_candidates` — pages that are already +/// free and observable by no snapshot, hence safe to overwrite — and only then +/// bump-allocated from `next_page`. A carved host is removed from the persisted +/// entries (it now holds the chain itself). The caller MUST ensure +/// `host_candidates` are a subset of `entries`' pages and disjoint from any +/// live page or the old chain's own pages (which must stay readable until the +/// header swap). +pub async fn rewrite_chain( + pager: &Pager, + realm_id: RealmId, + page_size: usize, + mut entries: Vec<(u64, u64)>, + host_candidates: Vec, + next_page: u64, +) -> Result<(u64, u64)> { + let cap = chain_capacity(page_size); + let total = entries.len(); + let mut next = next_page; + let mut carved: HashSet = HashSet::new(); + let mut chain_pages: Vec = Vec::new(); + let mut hosts = host_candidates.into_iter(); + loop { + let remaining = total - carved.len(); + let need = if remaining == 0 { + 0 + } else { + remaining.div_ceil(cap) + }; + if chain_pages.len() >= need { + break; + } + if let Some(h) = hosts.next() { + carved.insert(h); + chain_pages.push(h); + } else { + chain_pages.push(next); + next += 1; + } + } + entries.retain(|(_, pid)| !carved.contains(pid)); + let head = write_chain(pager, realm_id, page_size, &chain_pages, &entries).await?; + Ok((head, next)) +} + +/// Write `entries` across the supplied `chain_pages` (which must provide enough +/// capacity: `chain_pages.len() * chain_capacity(page_size) >= entries.len()`), +/// linking them into a chain. Returns the new head page id, or `0` when there +/// is nothing to write. The pages are inserted into the pager's dirty set; the +/// caller flushes and commits the header (carrying the returned head). +pub async fn write_chain( + pager: &Pager, + realm_id: RealmId, + page_size: usize, + chain_pages: &[u64], + entries: &[(u64, u64)], +) -> Result { + if entries.is_empty() { + return Ok(0); + } + let cap = chain_capacity(page_size); + let body_len = body_capacity(page_size); + debug_assert!(chain_pages.len() * cap >= entries.len()); + let mut written = 0usize; + for (i, &page_id) in chain_pages.iter().enumerate() { + let chunk = &entries[written..(written + cap).min(entries.len())]; + let next = chain_pages.get(i + 1).copied().unwrap_or(0); + // The last page used carries 0 as `next` even if more pages were + // supplied than needed (they go unused). + let next = if chunk.is_empty() { 0 } else { next }; + let mut body = vec![0u8; body_len]; + body[0..8].copy_from_slice(&next.to_le_bytes()); + let chunk_len = u32::try_from(chunk.len()) + .map_err(|_| PagedbError::Io(std::io::Error::other("free-list chunk_len overflow")))?; + body[8..12].copy_from_slice(&chunk_len.to_le_bytes()); + for (j, (cid, pid)) in chunk.iter().enumerate() { + let off = PAGE_HEADER_LEN + j * ENTRY_LEN; + body[off..off + 8].copy_from_slice(&cid.to_le_bytes()); + body[off + 8..off + 16].copy_from_slice(&pid.to_le_bytes()); + } + pager + .write_main_page(page_id, realm_id, PageKind::Free, &body) + .await?; + written += chunk.len(); + if written >= entries.len() { + break; + } + } + Ok(chain_pages[0]) +} diff --git a/src/pager/mod.rs b/src/pager/mod.rs index 4661180..ada2747 100644 --- a/src/pager/mod.rs +++ b/src/pager/mod.rs @@ -3,6 +3,7 @@ pub mod cache; pub mod core; pub mod format; +pub mod freelist; pub mod header; pub use cache::PageCache; From 27d33d7a8b3d028e4d2de8b4b1955b282a66ec76 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 15 Jun 2026 05:33:14 +0800 Subject: [PATCH 2/8] feat(txn/db): wire durable free-list chain into open, commit, and recovery paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track `free_list_root_page_id` in `WriterState` and decode it from the A/B header on open. Every header-writing code path (commit, rekey, segment link/unlink, reader-pin cleanup, compaction) now forwards the current chain head through `HeaderFieldsParams` so the slot is preserved across header swaps instead of being zero-filled. Key additions: `Db::free_page_consumed` — a per-txn sink (`Arc>>`) that records page ids drawn from `free_page_cache`. The commit path uses this to remove consumed pages from the durable chain (they now hold live data). Cleared at `begin_write` so each txn starts with a clean slate. `BTree::set_free_page_consumed` / `free_page_consumed` field — mirrors the existing `free_page_cache` hook; every page popped from the cache during `allocate_page` is pushed here so the commit path knows which chain entries to retire. `BTree::first_key` — leftmost-spine descent returning the smallest key in O(height). Used by `oldest_retained_history_commit` to find the oldest entry in the commit-history tree without a full scan. `Db::oldest_retained_history_commit` — computes the older of the in-memory reader floor and the oldest retained history commit; together these are the reclamation floor gating which free-list entries are safe to recycle at the start of a new write txn. `cleanup_stale_reader_pins` is strengthened to remove *all* durable pin rows on writer open (not just own-PID or wall-clock-expired ones). While the exclusive writer lock is held at open, no live reader can own a pin row yet, so every row present is a prior-incarnation leftover. The old heuristic could leave a crashed foreign-PID pin in place until its lease expired, permanently blocking the reclamation floor. `recovery::deep_walk` now walks and validates the free-list chain: verifies each chain page's AEAD, checks entries are in range, unique, and not simultaneously live, then accounts chain pages and tracked free pages as reachable so they are not reported as orphans. `DbMode` is threaded through `open_existing_inner` so open-time recovery (pin cleanup, backlog drain) runs only for Standalone / Follower handles that hold the exclusive writer lock. --- src/btree/tree/core.rs | 82 ++++++++++---------- src/btree/tree/scan.rs | 14 ++++ src/recovery/deep_walk.rs | 49 +++++++++++- src/txn/db/catalog.rs | 64 ++++++++++++++-- src/txn/db/core.rs | 45 +++++++++-- src/txn/db/misc.rs | 38 +++------- src/txn/db/mod.rs | 4 +- src/txn/db/open.rs | 76 ++++++++++++------- src/txn/db/rekey.rs | 3 + src/txn/db/segment.rs | 1 + src/txn/db/util.rs | 155 ++++++++++++++++++++++++++++++-------- src/txn/read.rs | 4 +- 12 files changed, 391 insertions(+), 144 deletions(-) diff --git a/src/btree/tree/core.rs b/src/btree/tree/core.rs index 019fd96..720da36 100644 --- a/src/btree/tree/core.rs +++ b/src/btree/tree/core.rs @@ -52,14 +52,18 @@ pub struct BTree { /// parallel-AEAD flush. In-place mutation by subsequent puts targeting /// the same leaf is allowed; no further allocation needed. pub(super) fresh_leaves: HashMap, - /// Cross-commit pool of pre-vetted reusable page IDs, shared (via `Arc`) - /// across all `BTree`s opened by the same `Db`. Allocation pops from - /// here before bumping `next_page_id`, keeping the file size bounded - /// when `OpenOptions::skip_freelist_persistence_when_no_readers` orphans - /// would otherwise accumulate. `None` for callers that haven't wired a - /// shared cache (compaction, history tree) — those keep bump-only - /// allocation. + /// Cross-commit pool of reusable page IDs, shared (via `Arc`) across the + /// main, catalog, and history `BTree`s of one `Db`. Allocation pops from + /// here before bumping `next_page_id`, recycling pages freed by earlier + /// commits (once no reader or retained-history root can observe them) so + /// the file stays bounded under sustained writes. `None` for callers that + /// haven't wired a shared cache (e.g. compaction's repack trees), which + /// keep bump-only allocation. pub(super) free_page_cache: Option>>>, + /// Sink recording page ids drawn from `free_page_cache` and reused this + /// session. The commit path removes them from the durable free-list (they + /// now hold live committed data). Shared (via `Arc`) across the txn's trees. + pub(super) free_page_consumed: Option>>>, /// Last key successfully appended via [`Self::put_append`]. Used to /// enforce the monotonic-key invariant on subsequent calls and to /// invalidate the cached path when any non-append mutation (regular @@ -96,6 +100,7 @@ impl BTree { dirty_parent_paths: HashMap::new(), fresh_leaves: HashMap::new(), free_page_cache: None, + free_page_consumed: None, append_last_key: None, append_cached_path: None, } @@ -110,22 +115,18 @@ impl BTree { } /// Wire in the `Db`'s shared free-page cache. After this call, - /// `allocate_page` will pop from the shared pool before bumping - /// `next_page_id`. Pages pushed into the pool by an earlier writer - /// commit (via [`Self::push_to_shared_cache`]) become reusable here. + /// `allocate_page` pops from the shared pool before bumping `next_page_id`. + /// The pool is loaded at `begin_write` with the durable free-list's + /// floor-safe pages, so recycling from it is always snapshot-safe. pub fn set_free_page_cache(&mut self, cache: Arc>>) { self.free_page_cache = Some(cache); } - /// Push `page_ids` into the shared free-page cache, if one is wired. - /// Used by the writer commit path when the no-reader fast-free option - /// is active: instead of orphaning freed pages, hand them to the next - /// txn's allocator. - pub fn push_to_shared_cache(&self, page_ids: &[u64]) { - if let Some(cache) = &self.free_page_cache { - let mut guard = cache.lock(); - guard.extend(page_ids.iter().copied()); - } + /// Wire the shared sink that records cache pages reused this session, so the + /// commit path can remove them from the durable free-list. Set alongside + /// [`Self::set_free_page_cache`]. + pub fn set_free_page_consumed(&mut self, consumed: Arc>>) { + self.free_page_consumed = Some(consumed); } #[must_use] @@ -148,34 +149,31 @@ impl BTree { } pub(super) fn allocate_page(&mut self) -> u64 { - // Reuse a freed page only if it is at or above the reuse threshold. - // Pages below the threshold may still be live in pinned reader snapshots. + // First, recycle a page freed earlier *in this same session*, gated by + // the reuse threshold: a page below it may still be live in a pinned + // reader's snapshot, so it can't be reused until the durable free-list + // clears it (it leaves via `drain_freed` at commit instead). if self.reuse_threshold == 0 { - // No readers pinned: recycle freely. if let Some(id) = self.freed.pop() { return id; } - // Consult the cross-commit shared cache (pages handed off by - // earlier writer commits under the no-reader fast-free option). - // Only safe to draw from this when no readers are pinned — the - // cache contract is "always safe to immediately reuse." - if let Some(cache) = &self.free_page_cache { - if let Some(id) = cache.lock().pop() { - return id; + } else if let Some(pos) = self + .freed + .iter() + .rposition(|&id| id >= self.reuse_threshold) + { + return self.freed.remove(pos); + } + // Then draw from the shared cross-commit cache. It is loaded at txn + // begin with *only* free-list pages below the reclamation floor — pages + // no live reader and no retained-history root can observe — so reusing + // them is safe regardless of `reuse_threshold`. Record each draw so the + // commit path removes it from the durable free-list. + if let Some(cache) = &self.free_page_cache { + if let Some(id) = cache.lock().pop() { + if let Some(consumed) = &self.free_page_consumed { + consumed.lock().push(id); } - } - } else { - // Readers pinned: only recycle pages that were allocated during - // this session (>= reuse_threshold) and thus cannot be in any - // prior snapshot. The shared cache is bypassed here because - // pages in it predate this session and may be visible to a - // pinned reader at an older commit. - if let Some(pos) = self - .freed - .iter() - .rposition(|&id| id >= self.reuse_threshold) - { - let id = self.freed.remove(pos); return id; } } diff --git a/src/btree/tree/scan.rs b/src/btree/tree/scan.rs index 9880332..fa8b6f0 100644 --- a/src/btree/tree/scan.rs +++ b/src/btree/tree/scan.rs @@ -38,6 +38,20 @@ impl BTree { } } + /// Return the smallest key in the tree, or `None` if the tree is empty. + /// Descends the leftmost spine only — O(tree height), not O(tree size). + pub async fn first_key(&self) -> Result>> { + if self.root_page_id == 0 { + return Ok(None); + } + // The empty key sorts below every stored key, so the descent lands on + // the leftmost leaf. + let path = self.path_to_leaf_for_key(&[]).await?; + let leaf_id = *path.last().expect("non-empty path"); + let leaf = self.read_leaf(leaf_id).await?; + Ok(leaf.records.first().map(|(k, _)| k.clone())) + } + /// Reverse range scan: `start` inclusive, `end` exclusive. Returns results /// in descending key order. Collects matching records forward then reverses. pub async fn scan_rev(&self, start: &[u8], end: &[u8]) -> Result, Vec)>> { diff --git a/src/recovery/deep_walk.rs b/src/recovery/deep_walk.rs index a9707dc..a0f30c8 100644 --- a/src/recovery/deep_walk.rs +++ b/src/recovery/deep_walk.rs @@ -146,12 +146,13 @@ impl DeepWalkReport { pub async fn run_deep_walk(db: &Db) -> Result { let mut report = DeepWalkReport::default(); - let (next_page_id, catalog_root, catalog_next) = { + let (next_page_id, catalog_root, catalog_next, free_list_root) = { let state = db.writer.lock().await; ( state.next_page_id, state.catalog_root_page_id, state.next_page_id, + state.free_list_root_page_id, ) }; @@ -163,7 +164,51 @@ pub async fn run_deep_walk(db: &Db) -> Result let realm_id = db.realm_id; // Collect the set of page IDs reachable from all live roots. - let reachable = collect_reachable_pages(db).await; + let mut reachable = collect_reachable_pages(db).await; + + // Walk and validate the durable free-list chain. Reading it verifies each + // chain page's AEAD; a corrupt chain surfaces as a page issue. Validate the + // entries (in range, unique, and not also live), then account the chain's + // own pages and the free pages it tracks so they are not reported as + // orphans — they are legitimately owned by the free-list, not stray. + match crate::pager::freelist::read_chain(&db.pager, realm_id, free_list_root).await { + Ok((free_entries, chain_pages)) => { + let mut seen: BTreeSet = BTreeSet::new(); + for &(_cid, pid) in &free_entries { + if pid >= next_page_id { + report.page_issues.push(PageIssue { + page_id: pid, + description: "free-list entry references a page past next_page_id" + .to_string(), + }); + } else if reachable.contains(&pid) { + report.page_issues.push(PageIssue { + page_id: pid, + description: "page is both live (reachable from a root) and free-listed" + .to_string(), + }); + } + if !seen.insert(pid) { + report.page_issues.push(PageIssue { + page_id: pid, + description: "duplicate free-list entry".to_string(), + }); + } + } + for p in chain_pages { + reachable.insert(p); + } + for (_cid, pid) in free_entries { + reachable.insert(pid); + } + } + Err(e) => { + report.page_issues.push(PageIssue { + page_id: free_list_root, + description: format!("free-list chain unreadable: {e}"), + }); + } + } let vfs: &V = &db.vfs; let main_file_res = vfs.open(main_db_path, OpenMode::Read).await; diff --git a/src/txn/db/catalog.rs b/src/txn/db/catalog.rs index 536fbd0..efa4cd2 100644 --- a/src/txn/db/catalog.rs +++ b/src/txn/db/catalog.rs @@ -15,6 +15,42 @@ use super::core::{ }; impl Db { + /// The oldest commit id still retained in the commit-history index, or + /// `None` when history is disabled or the index is empty. Pages reachable + /// from this commit's root (or any newer one) must not be recycled, so it + /// is one of the two floors gating free-page reclamation (the other is the + /// oldest live reader pin). Reads only the leftmost spine of the history + /// tree — O(height), not O(retained count). + pub(crate) async fn oldest_retained_history_commit( + &self, + commit_history_root_page_id: u64, + next_page_id: u64, + ) -> Result> { + if matches!( + self.options.commit_history_retain, + crate::options::RetainPolicy::Disabled + ) || commit_history_root_page_id == 0 + { + return Ok(None); + } + let hist = BTree::open( + self.pager.clone(), + self.realm_id, + commit_history_root_page_id, + next_page_id, + self.page_size, + ); + let Some(key) = hist.first_key().await? else { + return Ok(None); + }; + if key.len() < 8 { + return Ok(None); + } + let mut b = [0u8; 8]; + b.copy_from_slice(&key[..8]); + Ok(Some(u64::from_be_bytes(b))) + } + /// Scan catalog counter rows and bump any whose stored value is less than /// `anchor` up to `anchor`. Called once at open to recover monotonicity /// after a torn write. Errors are silently ignored (best-effort); a failure @@ -87,6 +123,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: state.commit_history_root_page_id, commit_history_root_version: state.commit_history_root_version, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; let hk_clone = self.hk.read().clone(); @@ -138,6 +175,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; let hk_clone = { self.hk.read().clone() }; @@ -180,16 +218,17 @@ impl Db { } } - /// Insert a commit-history entry into the commit-history B+ tree, prune - /// according to the retention policy, and return the updated - /// `(root_page_id, root_version, new_next_page_id)`. + /// Insert the new commit-history entry and prune per the retention policy. + /// Returns the page ids freed by this tree's copy-on-write and pruning, so + /// the caller can hand them to the shared allocator cache for reuse (they + /// are never reader-pinned). #[allow(clippy::too_many_lines)] pub(crate) async fn write_commit_history_entry( &self, state: &mut WriterState, new_commit_id: u64, meta: CommitHistoryMeta, - ) -> Result<()> { + ) -> Result> { let min_pinned = { let readers = self.tracked_readers.lock(); readers.iter().map(|r| r.commit_id.0).min() @@ -202,6 +241,14 @@ impl Db { state.next_page_id, self.page_size, ); + // The commit-history tree is not part of any reader's pinned snapshot + // (readers track the data and catalog roots, never the history root), so + // every page its copy-on-write/prune frees is immediately reusable. + // Recycle freely and feed the shared allocator cache so per-commit + // history churn does not leak pages over a long-lived writer's lifetime. + hist_tree.set_reuse_threshold(0); + hist_tree.set_free_page_cache(self.free_page_cache.clone()); + hist_tree.set_free_page_consumed(self.free_page_consumed.clone()); // Insert the new entry. let key = new_commit_id.to_be_bytes().to_vec(); @@ -322,6 +369,13 @@ impl Db { // commit's unified `pager.flush_main` picks them up) without issuing a // separate fsync. The caller is responsible for flushing the pager. hist_tree.materialize_dirty().await?; + // Capture spine/prune frees after materialization (they are realized + // during the flush, not before it). + let freed: Vec = hist_tree + .drain_freed() + .into_iter() + .filter(|&p| p >= 4) + .collect(); let new_hist_root = hist_tree.root_page_id(); let new_next = hist_tree.next_page_id().max(state.next_page_id); @@ -329,6 +383,6 @@ impl Db { state.commit_history_root_version = new_commit_id; state.next_page_id = new_next; - Ok(()) + Ok(freed) } } diff --git a/src/txn/db/core.rs b/src/txn/db/core.rs index 7930dff..22d830e 100644 --- a/src/txn/db/core.rs +++ b/src/txn/db/core.rs @@ -48,6 +48,10 @@ pub(crate) struct WriterState { pub seq: u64, pub catalog_root_page_id: u64, pub catalog_root_txn_id: u64, + /// Head page id of the durable free-list chain (0 = empty). Stored in the + /// A/B header's `free_list_root` slot and rewritten atomically with each + /// commit. See [`crate::pager::freelist`]. + pub free_list_root_page_id: u64, /// Root page id of the commit-history B+ tree (0 = not yet created). pub commit_history_root_page_id: u64, /// Version / `txn_id` of the commit-history tree's last write. @@ -117,14 +121,18 @@ pub struct Db { /// `PinHandle` so durable-pin commit paths can also publish. pub(crate) snapshot: Arc>, /// Cross-commit cache of page IDs known to be safely reusable. Populated - /// on commit when `skip_freelist_persistence_when_no_readers` is enabled - /// and no readers were pinned: instead of orphaning freed pages, recycle - /// them in-memory so the next writer txn's `allocate_page` pops from - /// here before bumping `next_page_id`. Keeps the file size bounded under - /// the fast-free option. Shared with each session's `BTree` via the same - /// `Arc` so all three trees in a txn (main, catalog, history) draw from - /// the same pool. + /// after each commit with the pages it freed that no live reader and no + /// retained commit-history root can still observe; the next writer txn's + /// `allocate_page` pops from here before bumping `next_page_id`, keeping the + /// file size bounded under sustained writes. Shared with each session's + /// `BTree` via the same `Arc` so all three trees in a txn (main, catalog, + /// history) draw from the same pool. Cleared by `compact_now`'s full repack, + /// which relocates pages and invalidates every cached id. pub(crate) free_page_cache: Arc>>, + /// Per-txn sink (cleared at `begin_write`) recording page ids the allocator + /// drew from `free_page_cache`. The commit path removes them from the + /// durable free-list — they now hold live committed data. + pub(crate) free_page_consumed: Arc>>, } /// Reader-visible state, refreshed by the writer at commit time. @@ -146,6 +154,23 @@ pub(super) fn encode_root_ref(page_id: u64, txn_id: u64) -> [u8; 16] { bytes } +/// Encode the header's `free_list_root` slot from the free-list chain head page +/// id (low 8 bytes, LE; remaining bytes reserved/zero). +#[must_use] +pub(crate) fn encode_free_list_root(head_page_id: u64) -> [u8; 16] { + let mut bytes = [0u8; 16]; + bytes[..8].copy_from_slice(&head_page_id.to_le_bytes()); + bytes +} + +/// Decode the free-list chain head page id from the header's `free_list_root`. +#[must_use] +pub(crate) fn decode_free_list_root(raw: &[u8; 16]) -> u64 { + let mut b = [0u8; 8]; + b.copy_from_slice(&raw[..8]); + u64::from_le_bytes(b) +} + /// Variable fields supplied per call site when assembling a /// [`MainDbHeaderFields`] for a writer header commit. The constant fields /// (`format_version`, `flags`, identity bytes, and the always-zero @@ -163,6 +188,10 @@ pub(super) struct HeaderFieldsParams { pub commit_history_root_page_id: u64, pub commit_history_root_version: u64, pub next_page_id: u64, + /// Head page id of the durable free-list chain to record in the header. + /// Header writes that don't touch the free list pass the current + /// `state.free_list_root_page_id` so it is preserved across the swap. + pub free_list_root_page_id: u64, } impl Db { @@ -185,7 +214,7 @@ impl Db { active_root_txn_id: params.active_root_txn_id, counter_anchor: params.counter_anchor, commit_id: CommitId(params.commit_id), - free_list_root: [0; 16], + free_list_root: encode_free_list_root(params.free_list_root_page_id), catalog_root: params.catalog_root, apply_journal_root_page_id: 0, apply_journal_root_version: 0, diff --git a/src/txn/db/misc.rs b/src/txn/db/misc.rs index 11ff4b0..d9e5ae1 100644 --- a/src/txn/db/misc.rs +++ b/src/txn/db/misc.rs @@ -96,16 +96,23 @@ impl Db { use std::sync::atomic::Ordering as AtOrd; // Gather writer-guarded values. - let (latest_commit_id, next_page_id, catalog_root, catalog_next) = { + let (latest_commit_id, next_page_id, catalog_root, catalog_next, free_list_root) = { let w = self.writer.lock().await; ( w.latest_commit_id, w.next_page_id, w.catalog_root_page_id, w.next_page_id, + w.free_list_root_page_id, ) }; + // Durable free-list depth (chain rooted at the header's free_list_root). + let free_list_pending_entries = + crate::pager::freelist::read_chain(&self.pager, self.realm_id, free_list_root) + .await + .map_or(0, |(entries, _)| entries.len() as u64); + // Main database file size. let main_db_bytes = match self .vfs @@ -137,10 +144,9 @@ impl Db { let pending_tombstones = u32::try_from(self.pending_tombstones.lock().len()).unwrap_or(u32::MAX); - // Segment stats and free-list counts from catalog. - let (segments_live, segments_total_bytes, free_list_pending_entries) = if catalog_root == 0 - { - (0u32, 0u64, 0u64) + // Segment stats from catalog. + let (segments_live, segments_total_bytes) = if catalog_root == 0 { + (0u32, 0u64) } else { let tree = BTree::open( self.pager.clone(), @@ -150,7 +156,6 @@ impl Db { self.page_size, ); - // Segments. let seg_start = vec![CatalogRowKind::Segment as u8]; let mut seg_end = seg_start.clone(); seg_end.push(0xFF); @@ -165,26 +170,7 @@ impl Db { .map(|m| m.total_bytes) .sum(); - // Free-list entries (persistent free-list rows). - let fl_start = vec![CatalogRowKind::FreeList as u8]; - let mut fl_end = fl_start.clone(); - fl_end.push(0xFF); - let fl_rows = tree - .collect_range(&fl_start, &fl_end) - .await - .unwrap_or_default(); - let fl_count = fl_rows.len() as u64; - - // Deferred-free entries. - let df_key = Catalog::deferred_free_key(); - let df_count = match tree.get(&df_key).await { - Ok(Some(bytes)) => { - Catalog::decode_deferred_free(&bytes).map_or(0, |v| v.len() as u64) - } - _ => 0, - }; - - (seg_count, seg_bytes, fl_count + df_count) + (seg_count, seg_bytes) }; Ok(DbStats { diff --git a/src/txn/db/mod.rs b/src/txn/db/mod.rs index e1bc9c7..d42c85a 100644 --- a/src/txn/db/mod.rs +++ b/src/txn/db/mod.rs @@ -14,4 +14,6 @@ mod snapshot; mod util; pub use core::Db; -pub(crate) use core::{CommitHistoryMeta, PendingTombstone, ReaderSnapshot, WriterState}; +pub(crate) use core::{ + CommitHistoryMeta, PendingTombstone, ReaderSnapshot, WriterState, encode_free_list_root, +}; diff --git a/src/txn/db/open.rs b/src/txn/db/open.rs index 81efccd..7437502 100644 --- a/src/txn/db/open.rs +++ b/src/txn/db/open.rs @@ -152,6 +152,7 @@ impl Db { seq: 1, catalog_root_page_id: 0, catalog_root_txn_id: 0, + free_list_root_page_id: 0, commit_history_root_page_id: 0, commit_history_root_version: 0, // Fresh DB: the commit-history tree is empty. @@ -191,6 +192,7 @@ impl Db { catalog_root_page_id: 0, })), free_page_cache: Arc::new(parking_lot::Mutex::new(Vec::new())), + free_page_consumed: Arc::new(parking_lot::Mutex::new(Vec::new())), }) } @@ -202,7 +204,7 @@ impl Db { realm: RealmId, options: OpenOptions, ) -> Result { - Self::open_existing_inner(vfs, kek, page_size, realm, options).await + Self::open_existing_inner(vfs, kek, page_size, realm, options, DbMode::Standalone).await } /// Open an existing database that was previously created with @@ -214,7 +216,15 @@ impl Db { page_size: usize, realm: RealmId, ) -> Result { - Self::open_existing_inner(vfs, kek, page_size, realm, OpenOptions::default()).await + Self::open_existing_inner( + vfs, + kek, + page_size, + realm, + OpenOptions::default(), + DbMode::Standalone, + ) + .await } #[allow(clippy::too_many_lines)] @@ -224,6 +234,7 @@ impl Db { page_size: usize, realm: RealmId, options: OpenOptions, + mode: DbMode, ) -> Result { use crate::vfs::VfsFile; use crate::vfs::types::OpenMode; @@ -327,6 +338,7 @@ impl Db { seq: fields.seq, catalog_root_page_id, catalog_root_txn_id, + free_list_root_page_id: super::core::decode_free_list_root(&fields.free_list_root), commit_history_root_page_id: fields.commit_history_root_page_id, commit_history_root_version: fields.commit_history_root_version, // Lazily populated on first `write_commit_history_entry` call. @@ -384,7 +396,7 @@ impl Db { mmap_bytes_in_use: std::sync::Arc::new(AtomicU64::new(0)), spill_bytes_in_use: AtomicU64::new(0), txn_seq: AtomicU64::new(0), - mode: DbMode::Standalone, + mode, aborted_readers: parking_lot::Mutex::new(std::collections::HashSet::new()), sentinel_locks: Vec::new(), snapshot: Arc::new(parking_lot::RwLock::new(ReaderSnapshot { @@ -394,6 +406,7 @@ impl Db { catalog_root_page_id, })), free_page_cache: Arc::new(parking_lot::Mutex::new(Vec::new())), + free_page_consumed: Arc::new(parking_lot::Mutex::new(Vec::new())), }; // If an online rekey was interrupted (crash or abort), resume and @@ -407,29 +420,34 @@ impl Db { } } - // Remove stale and expired reader-pin rows: own-PID rows from a prior - // process incarnation (crash without cleanup) and rows whose - // expires_unix_seconds is in the past. - { - let mut state = db.writer.lock().await; - let hk_clone = db.hk.read().clone(); - let _ = cleanup_stale_reader_pins( - &db.pager, - &db.vfs, - &db.main_db_path, - &hk_clone, - db.realm_id, - db.page_size, - db.cipher_id, - db.file_id, - db.kek_salt, - db.mk_epoch.load(Ordering::SeqCst), - &mut state, - ) - .await; - // Snapshot may have advanced by the bulk-pin-cleanup commit - // performed inside cleanup_stale_reader_pins. - db.publish_snapshot(&state); + // Recovery work that mutates the catalog/header is only valid for + // handle modes that hold the exclusive writer lock. ReadOnly / Observer + // must never write, so they skip pin cleanup and backlog draining. + if matches!(mode, DbMode::Standalone | DbMode::Follower) { + // Remove every durable reader-pin row left behind by a prior + // (now-dead) incarnation. Safe to clear all of them while holding + // the exclusive writer lock — see `cleanup_stale_reader_pins`. + { + let mut state = db.writer.lock().await; + let hk_clone = db.hk.read().clone(); + let _ = cleanup_stale_reader_pins( + &db.pager, + &db.vfs, + &db.main_db_path, + &hk_clone, + db.realm_id, + db.page_size, + db.cipher_id, + db.file_id, + db.kek_salt, + db.mk_epoch.load(Ordering::SeqCst), + &mut state, + ) + .await; + // Snapshot may have advanced by the bulk-pin-cleanup commit + // performed inside cleanup_stale_reader_pins. + db.publish_snapshot(&state); + } } // Durable monotonicity recovery for named counters. @@ -564,9 +582,11 @@ impl Db { } } - // Delegate to the appropriate lower-level constructor. + // Delegate to the appropriate lower-level constructor. Pass `mode` + // through to the existing-DB path so open-time recovery (pin cleanup, + // backlog drain) runs only for writer-lock-holding modes. let mut db = if main_db_exists { - Self::open_existing_with_options(vfs, kek, page_size, realm, options).await? + Self::open_existing_inner(vfs, kek, page_size, realm, options, mode).await? } else { Self::open_internal_with_options(vfs, kek, page_size, realm, options).await? }; diff --git a/src/txn/db/rekey.rs b/src/txn/db/rekey.rs index 061e9aa..127c3ff 100644 --- a/src/txn/db/rekey.rs +++ b/src/txn/db/rekey.rs @@ -97,6 +97,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: state.next_page_id, })?; let new_slot = commit_header( @@ -336,6 +337,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; @@ -396,6 +398,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; diff --git a/src/txn/db/segment.rs b/src/txn/db/segment.rs index af8a251..fb5a308 100644 --- a/src/txn/db/segment.rs +++ b/src/txn/db/segment.rs @@ -250,6 +250,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; diff --git a/src/txn/db/util.rs b/src/txn/db/util.rs index f13f069..43f97a9 100644 --- a/src/txn/db/util.rs +++ b/src/txn/db/util.rs @@ -5,7 +5,6 @@ use std::sync::Arc; use crate::btree::BTree; -use crate::catalog::codec::Catalog; use crate::crypto::kdf::{derive_hk, derive_mk}; use crate::errors::PagedbError; use crate::pager::Pager; @@ -101,10 +100,24 @@ pub(super) fn next_lease_id() -> u64 { ts ^ (seq.wrapping_mul(0x9e37_79b9_7f4a_7c15)) } -/// Scan the catalog for durable reader-pin rows that are either stale (written -/// by the current PID from a previous process incarnation) or expired by wall -/// clock. Delete all such rows in a single bulk catalog commit. Called at -/// writer-open time to recover from reader crashes. +/// Delete **every** durable reader-pin row in the catalog in a single bulk +/// catalog commit. Called at writer-open time, and only for handle modes that +/// hold the exclusive writer lock (Standalone / Follower). +/// +/// Why deleting *all* pins is correct — not just own-PID or wall-clock-expired +/// ones: durable pin rows are written exclusively by a Standalone writer's +/// `begin_read` (other modes track readers in memory only). A Standalone / +/// Follower handle holds the exclusive writer lock for its entire lifetime, and +/// this cleanup runs while that lock is held at open, *before* any reader has +/// been handed out on the new handle. Therefore no live reader of the current +/// incarnation can own a pin row yet, and no other process can hold the writer +/// lock concurrently. Every pin row present at this moment is a leftover from a +/// prior, now-dead incarnation and is safe to remove. +/// +/// This supersedes the earlier own-PID-or-expired heuristic, which could not +/// remove a pin left by a *crashed* process running under a different PID until +/// its 30 s lease expired — and which, combined with `min_durable_reader_commit`, +/// let such a pin permanently block deferred-free drain and compaction. #[allow(clippy::too_many_arguments)] pub(crate) async fn cleanup_stale_reader_pins( pager: &Arc>, @@ -122,8 +135,6 @@ pub(crate) async fn cleanup_stale_reader_pins( if state.catalog_root_page_id == 0 { return Ok(()); } - let now = crate::txn::read::unix_now_seconds(); - let own_pid = std::process::id(); let tree = BTree::open( pager.clone(), realm_id, @@ -135,29 +146,10 @@ pub(crate) async fn cleanup_stale_reader_pins( let end = crate::catalog::codec::Catalog::reader_pin_range_end(); let rows = tree.collect_range(&start, &end).await?; - let stale_keys: Vec> = rows - .into_iter() - .filter_map(|(k, v)| { - // Key layout: [0x06] || pid_u32_be[4] || lease_id_u64_be[8] - if k.len() < 13 { - return Some(k); - } - let mut pid_buf = [0u8; 4]; - pid_buf.copy_from_slice(&k[1..5]); - let row_pid = u32::from_be_bytes(pid_buf); - // Own-PID rows from prior incarnation (crash without cleanup). - if row_pid == own_pid { - return Some(k); - } - // Expired rows. - if let Ok(pv) = Catalog::decode_reader_pin(&v) { - if pv.expires_unix_seconds < now { - return Some(k); - } - } - None - }) - .collect(); + // Every pin row present at writer open is a prior-incarnation leftover + // (see the function doc for the exclusive-writer-lock argument). Remove + // all of them. + let stale_keys: Vec> = rows.into_iter().map(|(k, _v)| k).collect(); if stale_keys.is_empty() { return Ok(()); @@ -193,7 +185,7 @@ pub(crate) async fn cleanup_stale_reader_pins( active_root_txn_id: state.latest_commit_id, counter_anchor, commit_id: CommitId(new_commit_id), - free_list_root: [0u8; 16], + free_list_root: super::core::encode_free_list_root(state.free_list_root_page_id), catalog_root: catalog_root_bytes, apply_journal_root_page_id: 0, apply_journal_root_version: 0, @@ -221,3 +213,104 @@ pub(crate) async fn cleanup_stale_reader_pins( state.seq = new_seq; Ok(()) } + +#[cfg(test)] +mod tests { + use std::sync::atomic::Ordering; + + use super::super::core::Db; + use crate::btree::BTree; + use crate::catalog::codec::Catalog; + use crate::txn::read::{insert_pin_row, make_pin_handle}; + use crate::vfs::memory::MemVfs; + use crate::{OpenOptions, RealmId}; + + /// Count the durable reader-pin rows currently in the catalog. + async fn pin_row_count(db: &Db) -> usize { + let (root, next) = { + let w = db.writer.lock().await; + (w.catalog_root_page_id, w.next_page_id) + }; + if root == 0 { + return 0; + } + let tree = BTree::open(db.pager.clone(), db.realm_id, root, next, db.page_size); + let start = Catalog::reader_pin_range_start(); + let end = Catalog::reader_pin_range_end(); + tree.collect_range(&start, &end).await.unwrap().len() + } + + /// A durable reader-pin row written by a *crashed* process running under a + /// different PID, with a lease far in the future, must be cleared the next + /// time a writer opens the store — otherwise it permanently pins an old + /// commit and blocks deferred-free drain / compaction. The old own-PID-or- + /// expired heuristic left such a row in place until its lease expired. + #[tokio::test(flavor = "current_thread")] + async fn open_clears_foreign_pid_reader_pins() { + let vfs = MemVfs::new(); + let kek = [3u8; 32]; + let realm = RealmId::new([9u8; 16]); + + // Create the store and a catalog root. + { + let db = Db::open(vfs.clone(), kek, 4096, realm, OpenOptions::default()) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + w.put(b"seed", b"v").await.unwrap(); + w.commit().await.unwrap(); + } + + // Inject a durable pin row owned by a foreign PID with a lease + // ~28 hours in the future (definitely not expired). + { + let mut state = db.writer.lock().await; + let foreign_pid = std::process::id().wrapping_add(0x5151); + let pin = make_pin_handle( + db.pager.clone(), + db.realm_id, + db.page_size, + db.main_db_path.clone(), + db.vfs.clone(), + db.hk.read().clone(), + db.mk_epoch.load(Ordering::SeqCst), + db.cipher_id, + db.file_id, + db.kek_salt, + db.latest_commit.load(Ordering::SeqCst), + db.snapshot.clone(), + foreign_pid, + 0x777, + 100_000, + ); + let (cid, root, cat) = ( + state.latest_commit_id, + state.root_page_id, + state.catalog_root_page_id, + ); + insert_pin_row(&pin, &mut state, cid, root, cat) + .await + .unwrap(); + } + + assert_eq!( + pin_row_count(&db).await, + 1, + "foreign-PID pin row should be present before reopen" + ); + // Drop the handle (releases the writer lock) without cleaning up — + // simulating the foreign writer's crash. + } + + // Reopen as a writer: open-time cleanup must remove the foreign pin. + let db2 = Db::open(vfs.clone(), kek, 4096, realm, OpenOptions::default()) + .await + .unwrap(); + assert_eq!( + pin_row_count(&db2).await, + 0, + "open must clear every durable reader-pin row left by a prior incarnation" + ); + } +} diff --git a/src/txn/read.rs b/src/txn/read.rs index 2b156a0..2b15236 100644 --- a/src/txn/read.rs +++ b/src/txn/read.rs @@ -411,7 +411,9 @@ fn make_header_fields( active_root_txn_id: state.latest_commit_id, counter_anchor, commit_id: crate::CommitId(new_commit_id), - free_list_root: [0u8; 16], + // Preserve the durable free-list head across pin-row commits, or every + // begin_read would wipe the free list. + free_list_root: crate::txn::db::encode_free_list_root(state.free_list_root_page_id), catalog_root: catalog_root_bytes, apply_journal_root_page_id: 0, apply_journal_root_version: 0, From 5b400c2254c26952202d6ab56ce3132acd94c520 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 15 Jun 2026 05:33:26 +0800 Subject: [PATCH 3/8] refactor(txn/write): split monolithic write.rs into focused submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single `src/txn/write.rs` (896 lines) grew to cover four distinct concerns. Replace it with `src/txn/write/` containing: - `txn.rs` — `WriteTxn` struct, `begin`/`abort`, and all mutation methods (`put`, `delete`, quota checks, segment ops) - `commit.rs` — the commit path: drain freed pages, rewrite the durable free-list chain, update the catalog/history trees, and swap the A/B header - `spill.rs` — `SpillScope` and `SpillSegmentMeta` (AEAD scratch file) - `counter.rs` — `CounterRef` (durable monotonic named counters) - `mod.rs` — re-exports only; no logic Public surface is unchanged (`WriteTxn`, `SpillScope`, `ScratchOffset`, `CounterRef` all re-export from `crate::txn::write`). The commit path in `commit.rs` is updated to use the durable free-list chain: at `begin_write` it reads the current chain to populate the in-memory allocator cache with floor-safe pages; at commit it rewrites the chain with newly freed pages added, consumed pages removed, and chain host pages recycled — all atomically with the header swap. --- src/txn/write.rs | 896 --------------------------------------- src/txn/write/commit.rs | 294 +++++++++++++ src/txn/write/counter.rs | 75 ++++ src/txn/write/mod.rs | 11 + src/txn/write/spill.rs | 249 +++++++++++ src/txn/write/txn.rs | 356 ++++++++++++++++ 6 files changed, 985 insertions(+), 896 deletions(-) delete mode 100644 src/txn/write.rs create mode 100644 src/txn/write/commit.rs create mode 100644 src/txn/write/counter.rs create mode 100644 src/txn/write/mod.rs create mode 100644 src/txn/write/spill.rs create mode 100644 src/txn/write/txn.rs diff --git a/src/txn/write.rs b/src/txn/write.rs deleted file mode 100644 index 554776a..0000000 --- a/src/txn/write.rs +++ /dev/null @@ -1,896 +0,0 @@ -//! `WriteTxn` — exclusive write session backed by a `CoW` B+ tree. On commit, -//! flushes dirty pages, writes the new A/B header, and advances the -//! visibility commit id. - -use std::sync::atomic::Ordering; - -use tracing; - -use tokio::sync::MutexGuard; - -use crate::btree::BTree; -use crate::catalog::codec::{Catalog, RealmQuotas, SegmentMeta}; -use crate::crypto::aad::{Aad, AadFields}; -use crate::crypto::cipher::Cipher; -use crate::crypto::kdf::derive_spill_key; -use crate::crypto::nonce::Nonce; -use crate::errors::{PagedbError, QuotaKind}; -use crate::pager::header::commit_header; -use crate::pager::structural_header::MainDbHeaderFields; -use crate::vfs::types::OpenMode; -use crate::vfs::{Vfs, VfsFile}; -use crate::{CommitId, RealmId, Result}; - -use super::db::{CommitHistoryMeta, Db, PendingTombstone, WriterState}; - -/// Opaque handle returned by [`SpillScope::append`]. Pass to -/// [`SpillScope::read`] to decrypt and retrieve the bytes. -#[derive(Debug, Clone, Copy)] -pub struct ScratchOffset(u64); - -/// Metadata for one ciphertext chunk in the per-txn spill scratch file. -/// Stored in memory only; the tmp file is discarded at commit/abort. -#[derive(Clone)] -pub(crate) struct SpillSegmentMeta { - /// Byte offset of this chunk (ciphertext body start) in the tmp file. - pub offset: u64, - /// Length of the original plaintext in bytes. - pub plaintext_len: u32, - /// Length of ciphertext body (without the 16-byte tag) in bytes. - /// Total on-disk size for this chunk = `ciphertext_len + 16`. - pub ciphertext_len: u32, - /// The 12-byte nonce used to encrypt this chunk, stored verbatim so we - /// can reconstruct a `Nonce` on read without an additional lookup. - pub nonce_bytes: [u8; 12], -} - -/// A borrowed view into an active `WriteTxn` that offers AEAD-encrypted spill -/// storage in a per-transaction tmp file. -/// -/// The tmp file lives at `tmp/scratch-`. It is created lazily on the -/// first `append` call and removed at `commit` or `abort` (best-effort). -/// -/// Nonce scheme: `txn_seq_le6 ‖ segment_index_le6`. -/// - First 6 bytes: the low 6 bytes of `txn_seq` in little-endian order. -/// - Last 6 bytes: the low 6 bytes of the per-append segment index in -/// little-endian order. -/// -/// This guarantees uniqueness across all appends within a txn and across -/// independent txns (different `txn_seq`), without requiring a durable -/// nonce anchor (the tmp file is discarded before any subsequent txn can -/// reuse the same `txn_seq`). -pub struct SpillScope<'scope, 'db, V: Vfs + Clone> { - txn: &'scope mut WriteTxn<'db, V>, -} - -impl SpillScope<'_, '_, V> { - /// Encrypt `bytes` under the per-txn spill key and append the ciphertext - /// plus 16-byte AEAD tag to the tmp file. Returns a `ScratchOffset` handle - /// usable with [`read`](Self::read). - /// - /// Returns `PagedbError::Quota { kind: QuotaKind::ScratchPages, … }` when - /// the cumulative ciphertext bytes (body + tag) would exceed the - /// `OpenOptions::scratch_bytes` budget. - pub async fn append(&mut self, bytes: &[u8]) -> Result { - let limit = self.txn.db.options.scratch_bytes as u64; - let segment_index = self.txn.spill_segments.len() as u64; - - // Extract immutable db fields before taking a mutable borrow for the - // cipher. Rust cannot prove these are disjoint borrows through the - // &mut reference, so we copy them out first. - let mk_epoch = self - .txn - .db - .mk_epoch - .load(std::sync::atomic::Ordering::SeqCst); - let realm_id = self.txn.db.realm_id; - let file_id = self.txn.db.file_id; - - // Build the per-append nonce: txn_seq_le6 || segment_index_le6. - let nonce = self.txn.next_spill_nonce(segment_index); - - // Derive the spill cipher lazily (mutably borrows txn, then releases). - self.txn.ensure_spill_cipher()?; - let cipher_id_byte = self - .txn - .spill_cipher - .as_ref() - .expect("just derived") - .id() - .as_byte(); - - // Build AAD: binds cipher_id, a spill sentinel page_kind (0xFE), - // mk_epoch, this segment's index (as page_id), realm_id, and file_id. - let aad = Aad::from_fields(AadFields { - cipher_id: cipher_id_byte, - page_kind: 0xFE, - mk_epoch, - page_id: segment_index, - realm_id, - segment_id: file_id, - }); - - let mut body = bytes.to_vec(); - let tag = self - .txn - .spill_cipher - .as_ref() - .expect("derived above") - .encrypt(&nonce, &aad, &mut body)?; - - // ciphertext body + 16-byte tag. - let pers_len = body.len() as u64 + 16; - let new_total = self.txn.spill_bytes_used.saturating_add(pers_len); - if new_total > limit { - return Err(PagedbError::quota( - self.txn.db.realm_id, - QuotaKind::ScratchPages, - new_total, - limit, - )); - } - - // Lazy: create tmp dir and file on first append. - let path = self.txn.ensure_spill_path().await?; - - let mut file = self.txn.db.vfs.open(&path, OpenMode::CreateOrOpen).await?; - let body_offset = self.txn.spill_bytes_used; - let tag_offset = body_offset + body.len() as u64; - file.write_at(body_offset, &body).await?; - file.write_at(tag_offset, &tag).await?; - file.sync().await?; - - let plaintext_len = u32::try_from(bytes.len()).map_err(|_| PagedbError::PayloadTooLarge)?; - let ciphertext_len = u32::try_from(body.len()).map_err(|_| PagedbError::PayloadTooLarge)?; - - self.txn.spill_segments.push(SpillSegmentMeta { - offset: body_offset, - plaintext_len, - ciphertext_len, - nonce_bytes: *nonce.as_bytes(), - }); - self.txn.spill_bytes_used = new_total; - self.txn - .db - .spill_bytes_in_use - .store(new_total, std::sync::atomic::Ordering::Relaxed); - - Ok(ScratchOffset(segment_index)) - } - - /// Decrypt and return the bytes previously written via - /// [`append`](Self::append) at `handle`. - pub async fn read(&self, handle: ScratchOffset) -> Result> { - let idx = usize::try_from(handle.0).map_err(|_| PagedbError::NotFound)?; - let meta = self - .txn - .spill_segments - .get(idx) - .ok_or(PagedbError::NotFound)? - .clone(); - let path = self.txn.spill_path.as_ref().ok_or(PagedbError::NotFound)?; - - let file = self.txn.db.vfs.open(path, OpenMode::Read).await?; - - let body_len = meta.ciphertext_len as usize; - let mut body = vec![0u8; body_len]; - let mut tag = [0u8; 16]; - file.read_at(meta.offset, &mut body).await?; - file.read_at(meta.offset + body_len as u64, &mut tag) - .await?; - - let cipher = self.txn.spill_cipher_readonly()?; - let nonce = Nonce::from_bytes(meta.nonce_bytes); - let aad = Aad::from_fields(AadFields { - cipher_id: cipher.id().as_byte(), - page_kind: 0xFE, - mk_epoch: self - .txn - .db - .mk_epoch - .load(std::sync::atomic::Ordering::SeqCst), - page_id: handle.0, - realm_id: self.txn.db.realm_id, - segment_id: self.txn.db.file_id, - }); - - cipher.decrypt(&nonce, &aad, &mut body, &tag)?; - body.truncate(meta.plaintext_len as usize); - Ok(body) - } -} - -/// A handle to a named durable monotonic counter, scoped to a `WriteTxn`. -/// -/// The counter starts at `0` if it has never been written. Values are stored -/// as 8-byte little-endian `u64` catalog rows with key prefix `0x02`. -/// -/// Monotonicity guarantee: `set(v)` returns `PagedbError::Aborted` when -/// `v < current`; `increment_by` can never produce a value smaller than the -/// current one. Values only survive once the enclosing `WriteTxn::commit` -/// succeeds. -pub struct CounterRef<'a, V: Vfs + Clone> { - catalog_tree: &'a mut BTree, - main_tree: &'a mut BTree, - key: Vec, -} - -impl CounterRef<'_, V> { - /// Return the current counter value, or `0` if no row exists yet. - pub async fn get(&self) -> Result { - match self.catalog_tree.get(&self.key).await? { - Some(v) => Catalog::decode_counter(&v), - None => Ok(0), - } - } - - /// Set the counter to `value`. Returns `PagedbError::Aborted` when - /// `value < current` (strict monotonicity). - pub async fn set(&mut self, value: u64) -> Result<()> { - let current = self.get().await?; - if value < current { - return Err(PagedbError::Aborted); - } - Self::sync_to(self.main_tree, self.catalog_tree); - let bytes = Catalog::encode_counter(value); - self.catalog_tree.put(&self.key, &bytes).await?; - Self::sync_from(self.catalog_tree, self.main_tree); - Ok(()) - } - - /// Increment the counter by `delta` and return the new value. Returns - /// `PagedbError::NonceCounterExhausted` on `u64` overflow. - pub async fn increment_by(&mut self, delta: u64) -> Result { - let current = self.get().await?; - let next = current - .checked_add(delta) - .ok_or(PagedbError::NonceCounterExhausted)?; - Self::sync_to(self.main_tree, self.catalog_tree); - let bytes = Catalog::encode_counter(next); - self.catalog_tree.put(&self.key, &bytes).await?; - Self::sync_from(self.catalog_tree, self.main_tree); - Ok(next) - } - - /// Advance the catalog allocator cursor to be at least as high as the - /// main tree's before a catalog write. - fn sync_to(main: &BTree, catalog: &mut BTree) { - let shared = main.next_page_id().max(catalog.next_page_id()); - catalog.set_next_page_id(shared); - } - - /// After a catalog write, propagate any advances back to the main tree. - fn sync_from(catalog: &BTree, main: &mut BTree) { - let c = catalog.next_page_id(); - if main.next_page_id() < c { - main.set_next_page_id(c); - } - } -} - -/// Deferred filesystem operation applied after the A/B header is durable. -pub(crate) enum SegmentSideEffect { - Promote { segment_id: [u8; 16] }, - Tombstone { segment_id: [u8; 16] }, -} - -/// An exclusive write transaction. At most one `WriteTxn` exists per `Db` at -/// any time — the writer mutex enforces this. Either `commit` or `abort` must -/// be called; if the `WriteTxn` is dropped without either, dirty pages are -/// silently discarded (equivalent to abort). -pub struct WriteTxn<'db, V: Vfs + Clone> { - db: &'db Db, - guard: MutexGuard<'db, WriterState>, - btree: BTree, - catalog_tree: BTree, - pending_segments: Vec, - committed_or_aborted: bool, - /// Monotonic per-txn sequence number; assigned at `begin` from - /// `Db::txn_seq.fetch_add(1, Relaxed) + 1`. The first `WriteTxn` on a - /// fresh Db gets `txn_seq == 1`, making `tmp/scratch-1` predictable in - /// tests. - pub(crate) txn_seq: u64, - /// Lazily derived spill cipher. `None` until the first `spill_scope` append. - pub(crate) spill_cipher: Option, - /// Lazily created path to the per-txn spill tmp file (`tmp/scratch-`). - pub(crate) spill_path: Option, - /// Cumulative bytes (ciphertext body + tag) written to the spill file. - pub(crate) spill_bytes_used: u64, - /// Per-append metadata used by `SpillScope::read` to reconstruct AAD/nonce. - pub(crate) spill_segments: Vec, -} - -impl<'db, V: Vfs + Clone> WriteTxn<'db, V> { - pub(crate) async fn begin(db: &'db Db) -> Result> { - let guard = db.writer.lock().await; - // If any reader is pinned, freed pages from prior snapshots must not be - // recycled within this txn. Set the reuse threshold to next_page_id so - // that only pages allocated in this session (and then freed within it) - // can be recycled. - let reuse_threshold = { - let readers = db.tracked_readers.lock(); - if readers.is_empty() { - 0 - } else { - guard.next_page_id - } - }; - let mut btree = BTree::open( - db.pager.clone(), - db.realm_id, - guard.root_page_id, - guard.next_page_id, - db.page_size, - ); - btree.set_reuse_threshold(reuse_threshold); - btree.set_free_page_cache(db.free_page_cache.clone()); - let mut catalog_tree = BTree::open( - db.pager.clone(), - db.realm_id, - guard.catalog_root_page_id, - guard.next_page_id, - db.page_size, - ); - catalog_tree.set_reuse_threshold(reuse_threshold); - catalog_tree.set_free_page_cache(db.free_page_cache.clone()); - // Assign a txn_seq starting from 1: fetch_add returns the old value (0 - // for the first call), so we add 1 to produce 1-based ids. - let txn_seq = db.txn_seq.fetch_add(1, Ordering::Relaxed) + 1; - Ok(Self { - db, - guard, - btree, - catalog_tree, - pending_segments: Vec::new(), - committed_or_aborted: false, - txn_seq, - spill_cipher: None, - spill_path: None, - spill_bytes_used: 0, - spill_segments: Vec::new(), - }) - } - - /// Return a [`SpillScope`] that writes AEAD-encrypted bytes to a per-txn - /// tmp file within the `scratch_bytes` budget from `OpenOptions`. - pub fn spill_scope(&mut self) -> SpillScope<'_, 'db, V> { - SpillScope { txn: self } - } - - /// Lazily derive and cache the per-txn spill cipher (AES-256-GCM keyed - /// with a spill-specific HKDF derivation from the master key). - pub(crate) fn ensure_spill_cipher(&mut self) -> Result<&Cipher> { - if self.spill_cipher.is_none() { - let pager_mk = self.db.pager.mk(); - let key = derive_spill_key(&pager_mk, &self.db.file_id, self.txn_seq)?; - self.spill_cipher = Some(Cipher::new_aes_gcm(&key)); - } - Ok(self.spill_cipher.as_ref().expect("just set")) - } - - /// Return a reference to the cached spill cipher for read paths (does NOT - /// lazily create — callers must have already called `ensure_spill_cipher` - /// or `append` at least once, which guarantees the cipher exists). - pub(crate) fn spill_cipher_readonly(&self) -> Result<&Cipher> { - self.spill_cipher.as_ref().ok_or(PagedbError::NotFound) - } - - /// Lazily create the `tmp/` directory and return the spill file path. - pub(crate) async fn ensure_spill_path(&mut self) -> Result { - if self.spill_path.is_none() { - self.db.vfs.mkdir_all("tmp").await?; - let path = format!("tmp/scratch-{}", self.txn_seq); - self.spill_path = Some(path); - } - Ok(self.spill_path.clone().expect("just set")) - } - - /// Build the per-append nonce deterministically from `(txn_seq, segment_index)`. - /// - /// Layout: `txn_seq_le6 ‖ segment_index_le6` (6 + 6 = 12 bytes). - /// Uniqueness: nonces are unique per-txn (distinct `segment_index`) and - /// across txns (distinct `txn_seq` in first 6 bytes). No durable anchor - /// is needed because the tmp file is discarded before any key reuse - /// could matter. - pub(crate) fn next_spill_nonce(&self, segment_index: u64) -> Nonce { - let mut bytes = [0u8; 12]; - let seq_le = self.txn_seq.to_le_bytes(); - let idx_le = segment_index.to_le_bytes(); - bytes[..6].copy_from_slice(&seq_le[..6]); - bytes[6..].copy_from_slice(&idx_le[..6]); - Nonce::from_bytes(bytes) - } - - /// Remove the spill tmp file if it was created. Errors are swallowed - /// (best-effort cleanup); the file is transient anyway. - pub(crate) async fn cleanup_spill_async(&mut self) { - if let Some(path) = self.spill_path.take() { - let _ = self.db.vfs.remove(&path).await; - } - } - - pub async fn get(&self, key: &[u8]) -> Result>> { - self.btree.get(key).await - } - - pub async fn put(&mut self, key: &[u8], value: &[u8]) -> Result<()> { - self.btree.put(key, value).await - } - - /// Append a key-value pair under the monotonic-key invariant. - /// - /// Subsequent calls within the same `WriteTxn` skip the - /// `path_to_leaf_for_key` descent by reusing the cached rightmost - /// path; splits and explicitly-invalidating operations (regular `put`, - /// `delete`) force a re-descent on the next call. - /// - /// Intended for op-logs, time-series indexes, FTS posting-list builds - /// — any workload where the embedder can guarantee monotonically - /// increasing keys. - /// - /// # Errors - /// - /// Returns [`PagedbError::AppendNotMonotonic`] if `key` is not - /// strictly greater than the previously-appended key in this txn. - pub async fn put_append(&mut self, key: &[u8], value: &[u8]) -> Result<()> { - self.btree.put_append(key, value).await - } - - pub async fn delete(&mut self, key: &[u8]) -> Result { - self.btree.delete(key).await - } - - pub async fn put_batch(&mut self, sorted: Vec<(Vec, Vec)>) -> Result<()> { - self.btree.put_batch(sorted).await - } - - pub async fn delete_batch(&mut self, sorted: Vec>) -> Result<()> { - self.btree.delete_batch(sorted).await - } - - pub async fn delete_range(&mut self, start: &[u8], end: &[u8]) -> Result { - self.btree.delete_range(start, end).await - } - - /// Return a `CounterRef` scoped to this transaction for the named counter. - /// - /// The returned handle borrows `self` mutably for its lifetime. Use the - /// counter, drop the handle, then continue with other transaction - /// operations. The name must be at most `MAX_SEGMENT_NAME_LEN` bytes - /// (`PagedbError::NameTooLong` otherwise). - pub fn counter<'tx>(&'tx mut self, name: &str) -> Result> { - let key = Catalog::counter_key(name.as_bytes())?; - Ok(CounterRef { - catalog_tree: &mut self.catalog_tree, - main_tree: &mut self.btree, - key, - }) - } - - /// Register a segment under `name` in the catalog and schedule promotion - /// of its staging file to the live path on commit. - pub async fn link_segment(&mut self, name: &str, meta: &SegmentMeta) -> Result<()> { - let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; - if self.catalog_tree.get(&key).await?.is_some() { - return Err(PagedbError::AlreadyLinked); - } - self.enforce_segment_bytes_quota(meta.realm_id, meta.total_bytes, 0) - .await?; - let value = Catalog::encode_segment_meta(meta); - self.sync_allocator_to_catalog(); - self.catalog_tree.put(&key, &value).await?; - self.sync_allocator_from_catalog(); - self.pending_segments.push(SegmentSideEffect::Promote { - segment_id: meta.segment_id, - }); - Ok(()) - } - - /// Remove the catalog row for `name` and schedule a tombstone rename on commit. - pub async fn unlink_segment(&mut self, name: &str) -> Result<()> { - let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; - let value = self - .catalog_tree - .get(&key) - .await? - .ok_or(PagedbError::NotLinked)?; - let meta = Catalog::decode_segment_meta(&value)?; - self.sync_allocator_to_catalog(); - let removed = self.catalog_tree.delete(&key).await?; - self.sync_allocator_from_catalog(); - if !removed { - return Err(PagedbError::NotLinked); - } - self.pending_segments.push(SegmentSideEffect::Tombstone { - segment_id: meta.segment_id, - }); - Ok(()) - } - - /// Atomically swap the segment recorded under `name`: tombstone the old - /// segment id and promote `new_meta`'s staging file on commit. - pub async fn replace_segment(&mut self, name: &str, new_meta: &SegmentMeta) -> Result<()> { - let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; - let existing = self - .catalog_tree - .get(&key) - .await? - .ok_or(PagedbError::NotLinked)?; - let old_meta = Catalog::decode_segment_meta(&existing)?; - self.enforce_segment_bytes_quota( - new_meta.realm_id, - new_meta.total_bytes, - old_meta.total_bytes, - ) - .await?; - let value = Catalog::encode_segment_meta(new_meta); - self.sync_allocator_to_catalog(); - self.catalog_tree.put(&key, &value).await?; - self.sync_allocator_from_catalog(); - self.pending_segments.push(SegmentSideEffect::Tombstone { - segment_id: old_meta.segment_id, - }); - self.pending_segments.push(SegmentSideEffect::Promote { - segment_id: new_meta.segment_id, - }); - Ok(()) - } - - /// Check if the realm's committed segment bytes plus `new_bytes` minus - /// `delta_remove_bytes` would exceed the configured cap. Returns `Ok(())` - /// when no cap is set or the projected total is within the limit. - async fn enforce_segment_bytes_quota( - &self, - realm: RealmId, - new_bytes: u64, - delta_remove_bytes: u64, - ) -> Result<()> { - let quota_key = Catalog::quota_key(realm); - let quotas = match self.catalog_tree.get("a_key).await? { - Some(v) => Catalog::decode_realm_quotas(&v)?, - None => RealmQuotas::default(), - }; - let Some(limit) = quotas.max_segment_bytes else { - return Ok(()); - }; - // Scan all catalog segment rows for this realm to sum committed bytes. - let mut prefix = Vec::with_capacity(17); - prefix.push(0x01u8); // CatalogRowKind::Segment - prefix.extend_from_slice(&realm.0); - let mut end = prefix.clone(); - end.push(0xFF); - let rows = self.catalog_tree.collect_range(&prefix, &end).await?; - let mut committed: u64 = 0; - for (_, v) in rows { - let meta = Catalog::decode_segment_meta(&v)?; - committed = committed.saturating_add(meta.total_bytes); - } - let after_remove = committed.saturating_sub(delta_remove_bytes); - let projected = after_remove.saturating_add(new_bytes); - if projected > limit { - return Err(PagedbError::quota( - realm, - QuotaKind::SegmentBytes, - projected, - limit, - )); - } - Ok(()) - } - - /// Flush dirty pages, write the A/B header, apply pending segment side - /// effects, and publish the new root to readers. Returns the assigned - /// `CommitId`. - #[allow(clippy::too_many_lines)] - pub async fn commit(mut self) -> Result { - let _span = tracing::debug_span!("txn.commit"); - // Note: span is not entered via `.entered()` to keep this async fn's - // future `Send`. Use `tracing::instrument` or enter in sync sections only. - // Compute commit_id early so we can tag deferred-free entries before - // the catalog flush. - let new_commit_id = self.guard.latest_commit_id + 1; - - // Pages freed in this commit that, on the fast-free path, will be - // pushed to the `Db`'s shared free-page cache *after* the header - // swap succeeds. Lifted out of the block below so the post-commit - // handoff can see it. - let mut fast_freed_for_cache: Vec = Vec::new(); - // Collect freed pages from both trees and add them to the - // deferred-free queue in the catalog (tagged with new_commit_id). - // This keeps the free-list persistent across reopen. - { - let freed_from_main = self.btree.drain_freed(); - let freed_from_catalog = self.catalog_tree.drain_freed(); - let all_freed: Vec = freed_from_main - .into_iter() - .chain(freed_from_catalog) - // Skip reserved pages (0..=3) — they must never enter the free-list. - .filter(|&pid| pid >= 4) - .collect(); - - // Track the deferred-free pair count for the stall-policy check. - // Populated below from `free_pages_deferred_batch`'s return value - // when we wrote pairs this commit; otherwise read once on demand. - let mut deferred_count: Option = None; - // Opt-in fast path: when the embedder has set - // `skip_freelist_persistence_when_no_readers` AND no readers are - // pinned at this commit, skip the catalog-tree CoW for the - // deferred-free row. Instead of orphaning the freed pages we - // route them into the `Db`'s shared `free_page_cache` further - // down (after the header swap is durable, so a mid-commit crash - // can't surface them prematurely). The next writer txn's - // allocator pops from there, keeping the file size bounded. - // This is a pagedb extension beyond the architecture spec, - // documented on - // `OpenOptions::skip_freelist_persistence_when_no_readers`. The - // default behavior (option `false`) is spec-conformant: every - // freed page is persisted to the deferred-free queue. - let skip_persist = self.db.options.skip_freelist_persistence_when_no_readers - && self.db.tracked_readers.lock().is_empty(); - // Stash the list for post-commit handoff to the shared cache. - // Cleared if `skip_persist` is false (pages go to the catalog - // queue instead) or empty. - if skip_persist { - fast_freed_for_cache = all_freed.clone(); - } - if !all_freed.is_empty() && !skip_persist { - self.sync_allocator_to_catalog(); - // Batch all freed pages into a single deferred-free put to - // avoid repeated CoW on the deferred-free row. - deferred_count = Some( - crate::compaction::freelist::free_pages_deferred_batch( - &mut self.catalog_tree, - new_commit_id, - &all_freed, - ) - .await?, - ); - self.sync_allocator_from_catalog(); - } - - // Evaluate the reader stall policy against the current deferred-free - // queue depth. On Reject or all-non-abortable AbortOldest this returns - // an error and the commit is aborted by the `?` propagation. - { - let count = if let Some(c) = deferred_count { - c - } else { - let dk = crate::catalog::codec::Catalog::deferred_free_key(); - match self.catalog_tree.get(&dk).await? { - Some(bytes) => (bytes.len() as u64) / 16, - None => 0, - } - }; - self.db.evaluate_stall_policy(count)?; - } - } - - // Materialize each tree's dirty-leaf cache into the pager WITHOUT - // fsyncing per tree. All three trees (main, catalog, history) share - // the same file/realm, so their dirty pages live in one pager dirty - // set — a single `pager.flush_main` below batches them into one - // fsync. `materialize_dirty` allocates new pages, so re-sync the - // shared allocator between trees. - self.btree.materialize_dirty().await?; - self.sync_allocator_to_catalog(); - self.catalog_tree.materialize_dirty().await?; - self.sync_allocator_from_catalog(); - let new_root = self.btree.root_page_id(); - let new_catalog_root = self.catalog_tree.root_page_id(); - let new_next_after_trees = self - .btree - .next_page_id() - .max(self.catalog_tree.next_page_id()); - - // Stage the guard's next_page_id so the history tree allocates from - // above the main/catalog pages. - self.guard.next_page_id = new_next_after_trees; - - // Timestamp for Age-based retention. - let unix_seconds = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - - let history_meta = CommitHistoryMeta { - active_root_page_id: new_root, - catalog_root_page_id: new_catalog_root, - free_list_root_page_id: 0, - next_page_id: new_next_after_trees, - unix_seconds, - }; - // Skip the entire commit-history tree maintenance when the policy is - // Disabled. Saves the per-commit history-tree CoW + flush chain. - if !matches!( - self.db.options.commit_history_retain, - crate::options::RetainPolicy::Disabled - ) { - self.db - .write_commit_history_entry(&mut self.guard, new_commit_id, history_meta) - .await?; - } - - // Single pager fsync for all three trees' dirty pages. Replaces the - // per-tree fsyncs that used to come from `btree.flush`, - // `catalog_tree.flush`, and `hist_tree.flush` — collapses three - // fsyncs into one before the header write. - self.db.pager.flush_main(self.db.realm_id).await?; - - let new_next = self.guard.next_page_id; - let new_seq = self.guard.seq + 1; - let counter_anchor = self.db.pager.pending_anchor(); - - let mut catalog_root_bytes = [0u8; 16]; - catalog_root_bytes[..8].copy_from_slice(&new_catalog_root.to_le_bytes()); - catalog_root_bytes[8..].copy_from_slice(&new_commit_id.to_le_bytes()); - - let (policy_tag, policy_value) = - encode_retain_policy(&self.db.options.commit_history_retain); - - let fields = MainDbHeaderFields { - format_version: 1, - cipher_id: self.db.cipher_id.as_byte(), - page_size_log2: page_size_log2(self.db.page_size)?, - flags: 0, - file_id: self.db.file_id, - kek_salt: self.db.kek_salt, - mk_epoch: self.db.mk_epoch.load(std::sync::atomic::Ordering::SeqCst), - seq: new_seq, - active_root_page_id: new_root, - active_root_txn_id: new_commit_id, - counter_anchor, - commit_id: CommitId(new_commit_id), - free_list_root: [0; 16], - catalog_root: catalog_root_bytes, - apply_journal_root_page_id: 0, - apply_journal_root_version: 0, - commit_history_root_page_id: self.guard.commit_history_root_page_id, - commit_history_root_version: self.guard.commit_history_root_version, - restore_mode: 0, - next_page_id: new_next, - commit_retain_policy_tag: policy_tag, - commit_retain_policy_value: policy_value, - }; - - let hk_clone = { self.db.hk.read().clone() }; - let new_slot = commit_header( - &*self.db.vfs, - &self.db.main_db_path, - &hk_clone, - &fields, - self.guard.active_slot, - self.db.page_size, - ) - .await?; - self.db.pager.commit_anchor(counter_anchor)?; - - // Apply pending segment side effects after the header is durable. - if !self.pending_segments.is_empty() { - self.db.vfs.mkdir_all("seg").await?; - self.db.vfs.sync_dir("seg").await?; - self.db.vfs.mkdir_all("seg/.tombstone").await?; - self.db.vfs.sync_dir("seg/.tombstone").await?; - for effect in &self.pending_segments { - match effect { - SegmentSideEffect::Promote { segment_id } => { - let staging = crate::segment::writer::staging_path(segment_id); - let live = crate::segment::writer::live_path(segment_id); - self.db.vfs.rename(&staging, &live).await?; - } - SegmentSideEffect::Tombstone { segment_id } => { - if self.db.segment_id_is_reader_pinned(*segment_id).await? { - self.db.pending_tombstones.lock().push(PendingTombstone { - segment_id: *segment_id, - commit_id: new_commit_id, - }); - } else { - let live = crate::segment::writer::live_path(segment_id); - let tomb = format!( - "seg/.tombstone/{}.{}", - crate::hex::to_hex_lower(segment_id), - new_commit_id, - ); - self.db.vfs.rename(&live, &tomb).await?; - } - } - } - } - self.db.vfs.sync_dir("seg").await?; - self.db.vfs.sync_dir("seg/.tombstone").await?; - } - - self.guard.root_page_id = new_root; - self.guard.next_page_id = new_next; - self.guard.active_slot = new_slot; - self.guard.seq = new_seq; - self.guard.latest_commit_id = new_commit_id; - self.guard.catalog_root_page_id = new_catalog_root; - self.guard.catalog_root_txn_id = new_commit_id; - // commit_history_root_page_id and commit_history_root_version are - // already updated inside write_commit_history_entry. - self.db.latest_commit.store(new_commit_id, Ordering::SeqCst); - self.committed_or_aborted = true; - - // Post-commit handoff to the shared free-page cache. Only happens - // on the opt-in fast-free path (`fast_freed_for_cache` is empty - // otherwise). Deferred to here so a mid-commit failure aborts - // before the pages become "available for reuse" — the still-active - // previous root may still reference them. - if !fast_freed_for_cache.is_empty() { - let mut cache = self.db.free_page_cache.lock(); - cache.extend(fast_freed_for_cache.iter().copied()); - } - - // Remove the spill tmp file (best-effort; error is non-fatal). - self.cleanup_spill_async().await; - - self.db - .spill_bytes_in_use - .store(0, std::sync::atomic::Ordering::Relaxed); - tracing::debug!( - name = "txn.commit", - commit_id = new_commit_id, - "write transaction committed" - ); - Ok(CommitId(new_commit_id)) - } - - /// Discard all in-flight dirty pages without writing anything durable. - /// The spill tmp file (if created) is removed before returning (best-effort; - /// errors are ignored). - pub async fn abort(mut self) { - tracing::debug!(name = "txn.abort", "write transaction aborted"); - self.db.pager.discard_dirty_main(self.db.realm_id); - self.committed_or_aborted = true; - self.db - .spill_bytes_in_use - .store(0, std::sync::atomic::Ordering::Relaxed); - self.cleanup_spill_async().await; - } - - /// Before a catalog operation: ensure the catalog tree's allocator cursor - /// is at least as high as the main tree's, so neither tree allocates the - /// same page id. - fn sync_allocator_to_catalog(&mut self) { - let main_next = self.btree.next_page_id(); - let cat_next = self.catalog_tree.next_page_id(); - let shared = main_next.max(cat_next); - self.catalog_tree.set_next_page_id(shared); - } - - /// After a catalog operation: propagate any catalog allocation advances - /// back to the main tree so subsequent main-tree allocations stay above. - fn sync_allocator_from_catalog(&mut self) { - let cat_next = self.catalog_tree.next_page_id(); - self.btree.set_next_page_id(cat_next); - } -} - -impl Drop for WriteTxn<'_, V> { - fn drop(&mut self) { - if !self.committed_or_aborted { - self.db.pager.discard_dirty_main(self.db.realm_id); - } - } -} - -fn page_size_log2(page_size: usize) -> Result { - match page_size { - 4096 => Ok(12), - 8192 => Ok(13), - 16384 => Ok(14), - 32768 => Ok(15), - 65536 => Ok(16), - _ => Err(PagedbError::Unsupported), - } -} - -/// Encode a `RetainPolicy` into the two header fields `(tag, value)`. -/// tag 0 = Count, tag 1 = Age (seconds), tag 2 = Unbounded, tag 3 = Disabled. -fn encode_retain_policy(policy: &crate::options::RetainPolicy) -> (u8, u64) { - match policy { - crate::options::RetainPolicy::Count(n) => (0, u64::from(*n)), - crate::options::RetainPolicy::Age(d) => (1, d.as_secs()), - crate::options::RetainPolicy::Unbounded => (2, 0), - crate::options::RetainPolicy::Disabled => (3, 0), - } -} diff --git a/src/txn/write/commit.rs b/src/txn/write/commit.rs new file mode 100644 index 0000000..6bdea41 --- /dev/null +++ b/src/txn/write/commit.rs @@ -0,0 +1,294 @@ +//! `WriteTxn::commit` — flush dirty pages, rewrite the durable free-list, +//! write the A/B header, apply pending segment side effects, and publish the +//! new root to readers. +//! +//! ## Free-page reclamation +//! +//! A B+ tree commit copies-on-write every modified leaf and its spine, freeing +//! the old pages. The set of free pages lives in a durable chain rooted at the +//! header's `free_list_root` (see [`crate::pager::freelist`]) — outside the +//! catalog tree, so maintaining it adds no catalog churn and survives an +//! unclean shutdown. +//! +//! Each commit, after materializing all three trees, rebuilds the chain: +//! `(free pages before) − (pages reused this commit) + (pages freed this +//! commit) + (the old chain's own pages)`. Every entry is tagged with the +//! commit that freed it; `begin` recycles only those below the reclamation +//! floor (observable by no reader and no retained-history root). The chain is +//! committed atomically with the header swap, so no freed page is ever lost as +//! an orphan, and a page is recycled only after the commit that overwrote it is +//! durable. +//! +//! Crash-safety of the rewrite: new chain pages are drawn only from pages that +//! were *already free* before this commit (or freshly bump-allocated) — never +//! from pages this commit freed (still live under the old root until the swap) +//! nor from the old chain's own pages (must stay readable until the swap). On a +//! crash before the header swap, the old `free_list_root` and everything it +//! references are intact. + +use std::collections::HashSet; +use std::sync::atomic::Ordering; + +use crate::errors::PagedbError; +use crate::pager::freelist; +use crate::pager::header::commit_header; +use crate::pager::structural_header::MainDbHeaderFields; +use crate::vfs::Vfs; +use crate::{CommitId, Result}; + +use super::super::db::{CommitHistoryMeta, PendingTombstone, encode_free_list_root}; +use super::txn::{SegmentSideEffect, WriteTxn}; + +impl WriteTxn<'_, V> { + /// Flush dirty pages, write the A/B header, apply pending segment side + /// effects, and publish the new root to readers. Returns the assigned + /// `CommitId`. + #[allow(clippy::too_many_lines)] + pub async fn commit(mut self) -> Result { + let _span = tracing::debug_span!("txn.commit"); + // Note: span is not entered via `.entered()` to keep this async fn's + // future `Send`. Use `tracing::instrument` or enter in sync sections only. + let new_commit_id = self.guard.latest_commit_id + 1; + + // ── Materialize all trees first ────────────────────────────────────── + // Done before accounting freed pages so every copy-on-write spine free + // (which is realized during the flush, not before it) is captured. + self.btree.materialize_dirty().await?; + self.sync_allocator_to_catalog(); + self.catalog_tree.materialize_dirty().await?; + self.sync_allocator_from_catalog(); + let new_root = self.btree.root_page_id(); + let new_catalog_root = self.catalog_tree.root_page_id(); + self.guard.next_page_id = self + .btree + .next_page_id() + .max(self.catalog_tree.next_page_id()); + + // Commit-history entry (also materialized here). Its frees are never + // reader-pinned, so they fold into the free-list like any other. + let unix_seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + let history_meta = CommitHistoryMeta { + active_root_page_id: new_root, + catalog_root_page_id: new_catalog_root, + free_list_root_page_id: self.guard.free_list_root_page_id, + next_page_id: self.guard.next_page_id, + unix_seconds, + }; + let mut hist_freed: Vec = Vec::new(); + if !matches!( + self.db.options.commit_history_retain, + crate::options::RetainPolicy::Disabled + ) { + hist_freed = self + .db + .write_commit_history_entry(&mut self.guard, new_commit_id, history_meta) + .await?; + } + + // ── Account freed and reused pages ─────────────────────────────────── + // Pages freed by this commit (now live under the *old* root until the + // header swaps). Reserved pages 0..=3 are header/spare slots. + let all_freed: Vec = self + .btree + .drain_freed() + .into_iter() + .chain(self.catalog_tree.drain_freed()) + .chain(hist_freed) + .filter(|&pid| pid >= 4) + .collect(); + // Pages the allocator reused from the cache this txn — they were free + // before, now hold live committed data, so they leave the free-list. + let consumed: HashSet = std::mem::take(&mut *self.db.free_page_consumed.lock()) + .into_iter() + .collect(); + + // All free pages after this commit: the loaded chain's entries minus + // the ones reused, kept with their original freeing-commit tag. + let prior_free: Vec<(u64, u64)> = std::mem::take(&mut self.free_set_loaded) + .into_iter() + .filter(|(_, pid)| !consumed.contains(pid)) + .collect(); + let old_chain: Vec = std::mem::take(&mut self.old_chain_pages); + + // Pages safe to *host* the rewritten chain: only those below the + // reclamation floor and not reused — i.e. the remaining contents of the + // allocator cache (loaded at begin with exactly the floor-safe pages). + // A free-list entry with `cid >= floor` is a page a pinned reader still + // sees (it was freed *after* that reader's snapshot), so it must never + // be overwritten — only carried forward as an entry. + let host_candidates: Vec = std::mem::take(&mut *self.db.free_page_cache.lock()) + .into_iter() + .filter(|pid| !consumed.contains(pid)) + .collect(); + + // Assemble the new free-list: prior-free pages (kept with their original + // freeing-commit tag) plus this commit's frees and the now-superseded + // old chain pages, both tagged with this commit. + let mut entries: Vec<(u64, u64)> = prior_free; + entries.extend(all_freed.iter().map(|&pid| (new_commit_id, pid))); + entries.extend(old_chain.iter().map(|&pid| (new_commit_id, pid))); + + // The reader-stall policy fires only on entries genuinely stuck behind + // a pin — those at/above the reclamation floor. Drainable entries (below + // it) are being recycled and must not count, or an inherited-but- + // drainable backlog would spuriously abort a reader on reopen. Evaluated + // before the chain write so a reject aborts cleanly. + let stuck = entries + .iter() + .filter(|(cid, _)| *cid >= self.reclaim_floor) + .count() as u64; + self.db.evaluate_stall_policy(stuck)?; + + // Rewrite the chain, hosting it on the floor-safe pages (the cache + // remainder) and bump-allocating only if those run out. + let (new_free_list_root, new_next_page) = freelist::rewrite_chain( + &self.db.pager, + self.db.realm_id, + self.db.page_size, + entries, + host_candidates, + self.guard.next_page_id, + ) + .await?; + self.guard.next_page_id = new_next_page; + + // ── Flush + header swap ────────────────────────────────────────────── + self.db.pager.flush_main(self.db.realm_id).await?; + + let new_next = self.guard.next_page_id; + let new_seq = self.guard.seq + 1; + let counter_anchor = self.db.pager.pending_anchor(); + + let mut catalog_root_bytes = [0u8; 16]; + catalog_root_bytes[..8].copy_from_slice(&new_catalog_root.to_le_bytes()); + catalog_root_bytes[8..].copy_from_slice(&new_commit_id.to_le_bytes()); + + let (policy_tag, policy_value) = + encode_retain_policy(&self.db.options.commit_history_retain); + + let fields = MainDbHeaderFields { + format_version: 1, + cipher_id: self.db.cipher_id.as_byte(), + page_size_log2: page_size_log2(self.db.page_size)?, + flags: 0, + file_id: self.db.file_id, + kek_salt: self.db.kek_salt, + mk_epoch: self.db.mk_epoch.load(std::sync::atomic::Ordering::SeqCst), + seq: new_seq, + active_root_page_id: new_root, + active_root_txn_id: new_commit_id, + counter_anchor, + commit_id: CommitId(new_commit_id), + free_list_root: encode_free_list_root(new_free_list_root), + catalog_root: catalog_root_bytes, + apply_journal_root_page_id: 0, + apply_journal_root_version: 0, + commit_history_root_page_id: self.guard.commit_history_root_page_id, + commit_history_root_version: self.guard.commit_history_root_version, + restore_mode: 0, + next_page_id: new_next, + commit_retain_policy_tag: policy_tag, + commit_retain_policy_value: policy_value, + }; + + let hk_clone = { self.db.hk.read().clone() }; + let new_slot = commit_header( + &*self.db.vfs, + &self.db.main_db_path, + &hk_clone, + &fields, + self.guard.active_slot, + self.db.page_size, + ) + .await?; + self.db.pager.commit_anchor(counter_anchor)?; + + // Apply pending segment side effects after the header is durable. + if !self.pending_segments.is_empty() { + self.db.vfs.mkdir_all("seg").await?; + self.db.vfs.sync_dir("seg").await?; + self.db.vfs.mkdir_all("seg/.tombstone").await?; + self.db.vfs.sync_dir("seg/.tombstone").await?; + for effect in &self.pending_segments { + match effect { + SegmentSideEffect::Promote { segment_id } => { + let staging = crate::segment::writer::staging_path(segment_id); + let live = crate::segment::writer::live_path(segment_id); + self.db.vfs.rename(&staging, &live).await?; + } + SegmentSideEffect::Tombstone { segment_id } => { + if self.db.segment_id_is_reader_pinned(*segment_id).await? { + self.db.pending_tombstones.lock().push(PendingTombstone { + segment_id: *segment_id, + commit_id: new_commit_id, + }); + } else { + let live = crate::segment::writer::live_path(segment_id); + let tomb = format!( + "seg/.tombstone/{}.{}", + crate::hex::to_hex_lower(segment_id), + new_commit_id, + ); + self.db.vfs.rename(&live, &tomb).await?; + } + } + } + } + self.db.vfs.sync_dir("seg").await?; + self.db.vfs.sync_dir("seg/.tombstone").await?; + } + + self.guard.root_page_id = new_root; + self.guard.next_page_id = new_next; + self.guard.active_slot = new_slot; + self.guard.seq = new_seq; + self.guard.latest_commit_id = new_commit_id; + self.guard.catalog_root_page_id = new_catalog_root; + self.guard.catalog_root_txn_id = new_commit_id; + self.guard.free_list_root_page_id = new_free_list_root; + // commit_history_root_page_id and commit_history_root_version are + // already updated inside write_commit_history_entry. + self.db.latest_commit.store(new_commit_id, Ordering::SeqCst); + self.committed_or_aborted = true; + + // The allocator cache is rebuilt from the durable free-list at the next + // `begin_write`, so nothing is handed off here. + + // Remove the spill tmp file (best-effort; error is non-fatal). + self.cleanup_spill_async().await; + + self.db + .spill_bytes_in_use + .store(0, std::sync::atomic::Ordering::Relaxed); + tracing::debug!( + name = "txn.commit", + commit_id = new_commit_id, + "write transaction committed" + ); + Ok(CommitId(new_commit_id)) + } +} + +fn page_size_log2(page_size: usize) -> Result { + match page_size { + 4096 => Ok(12), + 8192 => Ok(13), + 16384 => Ok(14), + 32768 => Ok(15), + 65536 => Ok(16), + _ => Err(PagedbError::Unsupported), + } +} + +/// Encode a `RetainPolicy` into the two header fields `(tag, value)`. +/// tag 0 = Count, tag 1 = Age (seconds), tag 2 = Unbounded, tag 3 = Disabled. +fn encode_retain_policy(policy: &crate::options::RetainPolicy) -> (u8, u64) { + match policy { + crate::options::RetainPolicy::Count(n) => (0, u64::from(*n)), + crate::options::RetainPolicy::Age(d) => (1, d.as_secs()), + crate::options::RetainPolicy::Unbounded => (2, 0), + crate::options::RetainPolicy::Disabled => (3, 0), + } +} diff --git a/src/txn/write/counter.rs b/src/txn/write/counter.rs new file mode 100644 index 0000000..64b92ed --- /dev/null +++ b/src/txn/write/counter.rs @@ -0,0 +1,75 @@ +//! `CounterRef` — durable monotonic named counters scoped to a `WriteTxn`. + +use crate::Result; +use crate::btree::BTree; +use crate::catalog::codec::Catalog; +use crate::errors::PagedbError; +use crate::vfs::Vfs; + +/// A handle to a named durable monotonic counter, scoped to a `WriteTxn`. +/// +/// The counter starts at `0` if it has never been written. Values are stored +/// as 8-byte little-endian `u64` catalog rows with key prefix `0x02`. +/// +/// Monotonicity guarantee: `set(v)` returns `PagedbError::Aborted` when +/// `v < current`; `increment_by` can never produce a value smaller than the +/// current one. Values only survive once the enclosing `WriteTxn::commit` +/// succeeds. +pub struct CounterRef<'a, V: Vfs + Clone> { + pub(super) catalog_tree: &'a mut BTree, + pub(super) main_tree: &'a mut BTree, + pub(super) key: Vec, +} + +impl CounterRef<'_, V> { + /// Return the current counter value, or `0` if no row exists yet. + pub async fn get(&self) -> Result { + match self.catalog_tree.get(&self.key).await? { + Some(v) => Catalog::decode_counter(&v), + None => Ok(0), + } + } + + /// Set the counter to `value`. Returns `PagedbError::Aborted` when + /// `value < current` (strict monotonicity). + pub async fn set(&mut self, value: u64) -> Result<()> { + let current = self.get().await?; + if value < current { + return Err(PagedbError::Aborted); + } + Self::sync_to(self.main_tree, self.catalog_tree); + let bytes = Catalog::encode_counter(value); + self.catalog_tree.put(&self.key, &bytes).await?; + Self::sync_from(self.catalog_tree, self.main_tree); + Ok(()) + } + + /// Increment the counter by `delta` and return the new value. Returns + /// `PagedbError::NonceCounterExhausted` on `u64` overflow. + pub async fn increment_by(&mut self, delta: u64) -> Result { + let current = self.get().await?; + let next = current + .checked_add(delta) + .ok_or(PagedbError::NonceCounterExhausted)?; + Self::sync_to(self.main_tree, self.catalog_tree); + let bytes = Catalog::encode_counter(next); + self.catalog_tree.put(&self.key, &bytes).await?; + Self::sync_from(self.catalog_tree, self.main_tree); + Ok(next) + } + + /// Advance the catalog allocator cursor to be at least as high as the + /// main tree's before a catalog write. + fn sync_to(main: &BTree, catalog: &mut BTree) { + let shared = main.next_page_id().max(catalog.next_page_id()); + catalog.set_next_page_id(shared); + } + + /// After a catalog write, propagate any advances back to the main tree. + fn sync_from(catalog: &BTree, main: &mut BTree) { + let c = catalog.next_page_id(); + if main.next_page_id() < c { + main.set_next_page_id(c); + } + } +} diff --git a/src/txn/write/mod.rs b/src/txn/write/mod.rs new file mode 100644 index 0000000..c6c630f --- /dev/null +++ b/src/txn/write/mod.rs @@ -0,0 +1,11 @@ +//! `WriteTxn` and its supporting types: the exclusive write session, spill +//! scratch storage, durable monotonic counters, and the commit path. + +mod commit; +mod counter; +mod spill; +mod txn; + +pub use counter::CounterRef; +pub use spill::{ScratchOffset, SpillScope}; +pub use txn::WriteTxn; diff --git a/src/txn/write/spill.rs b/src/txn/write/spill.rs new file mode 100644 index 0000000..20a9ec4 --- /dev/null +++ b/src/txn/write/spill.rs @@ -0,0 +1,249 @@ +//! Per-`WriteTxn` AEAD-encrypted spill scratch storage. + +use crate::Result; +use crate::crypto::aad::{Aad, AadFields}; +use crate::crypto::cipher::Cipher; +use crate::crypto::kdf::derive_spill_key; +use crate::crypto::nonce::Nonce; +use crate::errors::{PagedbError, QuotaKind}; +use crate::vfs::types::OpenMode; +use crate::vfs::{Vfs, VfsFile}; + +use super::txn::WriteTxn; + +/// Opaque handle returned by [`SpillScope::append`]. Pass to +/// [`SpillScope::read`] to decrypt and retrieve the bytes. +#[derive(Debug, Clone, Copy)] +pub struct ScratchOffset(u64); + +/// Metadata for one ciphertext chunk in the per-txn spill scratch file. +/// Stored in memory only; the tmp file is discarded at commit/abort. +#[derive(Clone)] +pub(crate) struct SpillSegmentMeta { + /// Byte offset of this chunk (ciphertext body start) in the tmp file. + pub offset: u64, + /// Length of the original plaintext in bytes. + pub plaintext_len: u32, + /// Length of ciphertext body (without the 16-byte tag) in bytes. + /// Total on-disk size for this chunk = `ciphertext_len + 16`. + pub ciphertext_len: u32, + /// The 12-byte nonce used to encrypt this chunk, stored verbatim so we + /// can reconstruct a `Nonce` on read without an additional lookup. + pub nonce_bytes: [u8; 12], +} + +/// A borrowed view into an active `WriteTxn` that offers AEAD-encrypted spill +/// storage in a per-transaction tmp file. +/// +/// The tmp file lives at `tmp/scratch-`. It is created lazily on the +/// first `append` call and removed at `commit` or `abort` (best-effort). +/// +/// Nonce scheme: `txn_seq_le6 ‖ segment_index_le6`. +/// - First 6 bytes: the low 6 bytes of `txn_seq` in little-endian order. +/// - Last 6 bytes: the low 6 bytes of the per-append segment index in +/// little-endian order. +/// +/// This guarantees uniqueness across all appends within a txn and across +/// independent txns (different `txn_seq`), without requiring a durable +/// nonce anchor (the tmp file is discarded before any subsequent txn can +/// reuse the same `txn_seq`). +pub struct SpillScope<'scope, 'db, V: Vfs + Clone> { + pub(super) txn: &'scope mut WriteTxn<'db, V>, +} + +impl SpillScope<'_, '_, V> { + /// Encrypt `bytes` under the per-txn spill key and append the ciphertext + /// plus 16-byte AEAD tag to the tmp file. Returns a `ScratchOffset` handle + /// usable with [`read`](Self::read). + /// + /// Returns `PagedbError::Quota { kind: QuotaKind::ScratchPages, … }` when + /// the cumulative ciphertext bytes (body + tag) would exceed the + /// `OpenOptions::scratch_bytes` budget. + pub async fn append(&mut self, bytes: &[u8]) -> Result { + let limit = self.txn.db.options.scratch_bytes as u64; + let segment_index = self.txn.spill_segments.len() as u64; + + // Extract immutable db fields before taking a mutable borrow for the + // cipher. Rust cannot prove these are disjoint borrows through the + // &mut reference, so we copy them out first. + let mk_epoch = self + .txn + .db + .mk_epoch + .load(std::sync::atomic::Ordering::SeqCst); + let realm_id = self.txn.db.realm_id; + let file_id = self.txn.db.file_id; + + // Build the per-append nonce: txn_seq_le6 || segment_index_le6. + let nonce = self.txn.next_spill_nonce(segment_index); + + // Derive the spill cipher lazily (mutably borrows txn, then releases). + self.txn.ensure_spill_cipher()?; + let cipher_id_byte = self + .txn + .spill_cipher + .as_ref() + .expect("just derived") + .id() + .as_byte(); + + // Build AAD: binds cipher_id, a spill sentinel page_kind (0xFE), + // mk_epoch, this segment's index (as page_id), realm_id, and file_id. + let aad = Aad::from_fields(AadFields { + cipher_id: cipher_id_byte, + page_kind: 0xFE, + mk_epoch, + page_id: segment_index, + realm_id, + segment_id: file_id, + }); + + let mut body = bytes.to_vec(); + let tag = self + .txn + .spill_cipher + .as_ref() + .expect("derived above") + .encrypt(&nonce, &aad, &mut body)?; + + // ciphertext body + 16-byte tag. + let pers_len = body.len() as u64 + 16; + let new_total = self.txn.spill_bytes_used.saturating_add(pers_len); + if new_total > limit { + return Err(PagedbError::quota( + self.txn.db.realm_id, + QuotaKind::ScratchPages, + new_total, + limit, + )); + } + + // Lazy: create tmp dir and file on first append. + let path = self.txn.ensure_spill_path().await?; + + let mut file = self.txn.db.vfs.open(&path, OpenMode::CreateOrOpen).await?; + let body_offset = self.txn.spill_bytes_used; + let tag_offset = body_offset + body.len() as u64; + file.write_at(body_offset, &body).await?; + file.write_at(tag_offset, &tag).await?; + file.sync().await?; + + let plaintext_len = u32::try_from(bytes.len()).map_err(|_| PagedbError::PayloadTooLarge)?; + let ciphertext_len = u32::try_from(body.len()).map_err(|_| PagedbError::PayloadTooLarge)?; + + self.txn.spill_segments.push(SpillSegmentMeta { + offset: body_offset, + plaintext_len, + ciphertext_len, + nonce_bytes: *nonce.as_bytes(), + }); + self.txn.spill_bytes_used = new_total; + self.txn + .db + .spill_bytes_in_use + .store(new_total, std::sync::atomic::Ordering::Relaxed); + + Ok(ScratchOffset(segment_index)) + } + + /// Decrypt and return the bytes previously written via + /// [`append`](Self::append) at `handle`. + pub async fn read(&self, handle: ScratchOffset) -> Result> { + let idx = usize::try_from(handle.0).map_err(|_| PagedbError::NotFound)?; + let meta = self + .txn + .spill_segments + .get(idx) + .ok_or(PagedbError::NotFound)? + .clone(); + let path = self.txn.spill_path.as_ref().ok_or(PagedbError::NotFound)?; + + let file = self.txn.db.vfs.open(path, OpenMode::Read).await?; + + let body_len = meta.ciphertext_len as usize; + let mut body = vec![0u8; body_len]; + let mut tag = [0u8; 16]; + file.read_at(meta.offset, &mut body).await?; + file.read_at(meta.offset + body_len as u64, &mut tag) + .await?; + + let cipher = self.txn.spill_cipher_readonly()?; + let nonce = Nonce::from_bytes(meta.nonce_bytes); + let aad = Aad::from_fields(AadFields { + cipher_id: cipher.id().as_byte(), + page_kind: 0xFE, + mk_epoch: self + .txn + .db + .mk_epoch + .load(std::sync::atomic::Ordering::SeqCst), + page_id: handle.0, + realm_id: self.txn.db.realm_id, + segment_id: self.txn.db.file_id, + }); + + cipher.decrypt(&nonce, &aad, &mut body, &tag)?; + body.truncate(meta.plaintext_len as usize); + Ok(body) + } +} + +impl<'db, V: Vfs + Clone> WriteTxn<'db, V> { + /// Return a [`SpillScope`] that writes AEAD-encrypted bytes to a per-txn + /// tmp file within the `scratch_bytes` budget from `OpenOptions`. + pub fn spill_scope(&mut self) -> SpillScope<'_, 'db, V> { + SpillScope { txn: self } + } + + /// Lazily derive and cache the per-txn spill cipher (AES-256-GCM keyed + /// with a spill-specific HKDF derivation from the master key). + pub(crate) fn ensure_spill_cipher(&mut self) -> Result<&Cipher> { + if self.spill_cipher.is_none() { + let pager_mk = self.db.pager.mk(); + let key = derive_spill_key(&pager_mk, &self.db.file_id, self.txn_seq)?; + self.spill_cipher = Some(Cipher::new_aes_gcm(&key)); + } + Ok(self.spill_cipher.as_ref().expect("just set")) + } + + /// Return a reference to the cached spill cipher for read paths (does NOT + /// lazily create — callers must have already called `ensure_spill_cipher` + /// or `append` at least once, which guarantees the cipher exists). + pub(crate) fn spill_cipher_readonly(&self) -> Result<&Cipher> { + self.spill_cipher.as_ref().ok_or(PagedbError::NotFound) + } + + /// Lazily create the `tmp/` directory and return the spill file path. + pub(crate) async fn ensure_spill_path(&mut self) -> Result { + if self.spill_path.is_none() { + self.db.vfs.mkdir_all("tmp").await?; + let path = format!("tmp/scratch-{}", self.txn_seq); + self.spill_path = Some(path); + } + Ok(self.spill_path.clone().expect("just set")) + } + + /// Build the per-append nonce deterministically from `(txn_seq, segment_index)`. + /// + /// Layout: `txn_seq_le6 ‖ segment_index_le6` (6 + 6 = 12 bytes). + /// Uniqueness: nonces are unique per-txn (distinct `segment_index`) and + /// across txns (distinct `txn_seq` in first 6 bytes). No durable anchor + /// is needed because the tmp file is discarded before any key reuse + /// could matter. + pub(crate) fn next_spill_nonce(&self, segment_index: u64) -> Nonce { + let mut bytes = [0u8; 12]; + let seq_le = self.txn_seq.to_le_bytes(); + let idx_le = segment_index.to_le_bytes(); + bytes[..6].copy_from_slice(&seq_le[..6]); + bytes[6..].copy_from_slice(&idx_le[..6]); + Nonce::from_bytes(bytes) + } + + /// Remove the spill tmp file if it was created. Errors are swallowed + /// (best-effort cleanup); the file is transient anyway. + pub(crate) async fn cleanup_spill_async(&mut self) { + if let Some(path) = self.spill_path.take() { + let _ = self.db.vfs.remove(&path).await; + } + } +} diff --git a/src/txn/write/txn.rs b/src/txn/write/txn.rs new file mode 100644 index 0000000..bafaff2 --- /dev/null +++ b/src/txn/write/txn.rs @@ -0,0 +1,356 @@ +//! `WriteTxn` — exclusive write session backed by a `CoW` B+ tree. On commit, +//! flushes dirty pages, writes the new A/B header, and advances the +//! visibility commit id. The commit body lives in [`super::commit`]. + +use std::sync::atomic::Ordering; + +use tokio::sync::MutexGuard; + +use crate::btree::BTree; +use crate::catalog::codec::{Catalog, RealmQuotas, SegmentMeta}; +use crate::crypto::cipher::Cipher; +use crate::errors::{PagedbError, QuotaKind}; +use crate::vfs::Vfs; +use crate::{RealmId, Result}; + +use super::super::db::{Db, WriterState}; +use super::counter::CounterRef; +use super::spill::SpillSegmentMeta; + +/// Deferred filesystem operation applied after the A/B header is durable. +pub(crate) enum SegmentSideEffect { + Promote { segment_id: [u8; 16] }, + Tombstone { segment_id: [u8; 16] }, +} + +/// An exclusive write transaction. At most one `WriteTxn` exists per `Db` at +/// any time — the writer mutex enforces this. Either `commit` or `abort` must +/// be called; if the `WriteTxn` is dropped without either, dirty pages are +/// silently discarded (equivalent to abort). +pub struct WriteTxn<'db, V: Vfs + Clone> { + pub(super) db: &'db Db, + pub(super) guard: MutexGuard<'db, WriterState>, + pub(super) btree: BTree, + pub(super) catalog_tree: BTree, + pub(super) pending_segments: Vec, + pub(super) committed_or_aborted: bool, + /// Monotonic per-txn sequence number; assigned at `begin` from + /// `Db::txn_seq.fetch_add(1, Relaxed) + 1`. The first `WriteTxn` on a + /// fresh Db gets `txn_seq == 1`, making `tmp/scratch-1` predictable in + /// tests. + pub(crate) txn_seq: u64, + /// Lazily derived spill cipher. `None` until the first `spill_scope` append. + pub(crate) spill_cipher: Option, + /// Lazily created path to the per-txn spill tmp file (`tmp/scratch-`). + pub(crate) spill_path: Option, + /// Cumulative bytes (ciphertext body + tag) written to the spill file. + pub(crate) spill_bytes_used: u64, + /// Per-append metadata used by `SpillScope::read` to reconstruct AAD/nonce. + pub(crate) spill_segments: Vec, + /// The durable free-list's `(commit_id, page_id)` entries as loaded at + /// begin. The commit path rewrites the chain from these (minus the pages + /// reused this txn, plus the pages freed this txn). + pub(crate) free_set_loaded: Vec<(u64, u64)>, + /// Page ids the old free-list chain occupied at begin. They become free + /// once this commit's new chain supersedes them. + pub(crate) old_chain_pages: Vec, + /// Reclamation floor at begin: free-list entries tagged below this are + /// drainable (no snapshot pins them). The reader-stall policy is evaluated + /// at commit against only the entries at/above it — the backlog genuinely + /// stuck behind a reader pin — not the drainable remainder. + pub(crate) reclaim_floor: u64, +} + +impl<'db, V: Vfs + Clone> WriteTxn<'db, V> { + pub(crate) async fn begin(db: &'db Db) -> Result> { + let guard = db.writer.lock().await; + + // Pages freed *within this txn* must not be recycled within it while a + // reader is pinned (they may still be live in a prior snapshot). The + // reuse threshold gates the per-txn freed list for that; the shared + // cache is gated separately (it holds only floor-safe pages — below). + let min_reader = { + let readers = db.tracked_readers.lock(); + readers.iter().map(|r| r.commit_id.0).min() + }; + let reuse_threshold = if min_reader.is_none() { + 0 + } else { + guard.next_page_id + }; + + // Load the durable free-list and rebuild the shared allocator cache with + // exactly the pages below the reclamation floor — the older of the + // oldest live-reader pin and the oldest retained commit-history root. + // Those are observable by no snapshot, so they are safe to recycle now. + let history_floor = db + .oldest_retained_history_commit(guard.commit_history_root_page_id, guard.next_page_id) + .await?; + let floor = min_reader + .unwrap_or(u64::MAX) + .min(history_floor.map_or(u64::MAX, |h| h.saturating_add(1))); + let (free_set_loaded, old_chain_pages) = crate::pager::freelist::read_chain( + &db.pager, + db.realm_id, + guard.free_list_root_page_id, + ) + .await?; + { + let mut cache = db.free_page_cache.lock(); + cache.clear(); + for (cid, pid) in &free_set_loaded { + if *cid < floor { + cache.push(*pid); + } + } + } + db.free_page_consumed.lock().clear(); + + let mut btree = BTree::open( + db.pager.clone(), + db.realm_id, + guard.root_page_id, + guard.next_page_id, + db.page_size, + ); + btree.set_reuse_threshold(reuse_threshold); + btree.set_free_page_cache(db.free_page_cache.clone()); + btree.set_free_page_consumed(db.free_page_consumed.clone()); + let mut catalog_tree = BTree::open( + db.pager.clone(), + db.realm_id, + guard.catalog_root_page_id, + guard.next_page_id, + db.page_size, + ); + catalog_tree.set_reuse_threshold(reuse_threshold); + catalog_tree.set_free_page_cache(db.free_page_cache.clone()); + catalog_tree.set_free_page_consumed(db.free_page_consumed.clone()); + // Assign a txn_seq starting from 1: fetch_add returns the old value (0 + // for the first call), so we add 1 to produce 1-based ids. + let txn_seq = db.txn_seq.fetch_add(1, Ordering::Relaxed) + 1; + Ok(Self { + db, + guard, + btree, + catalog_tree, + pending_segments: Vec::new(), + committed_or_aborted: false, + txn_seq, + spill_cipher: None, + spill_path: None, + spill_bytes_used: 0, + spill_segments: Vec::new(), + free_set_loaded, + old_chain_pages, + reclaim_floor: floor, + }) + } + + pub async fn get(&self, key: &[u8]) -> Result>> { + self.btree.get(key).await + } + + pub async fn put(&mut self, key: &[u8], value: &[u8]) -> Result<()> { + self.btree.put(key, value).await + } + + /// Append a key-value pair under the monotonic-key invariant. + /// + /// Subsequent calls within the same `WriteTxn` skip the + /// `path_to_leaf_for_key` descent by reusing the cached rightmost + /// path; splits and explicitly-invalidating operations (regular `put`, + /// `delete`) force a re-descent on the next call. + /// + /// Intended for op-logs, time-series indexes, FTS posting-list builds + /// — any workload where the embedder can guarantee monotonically + /// increasing keys. + /// + /// # Errors + /// + /// Returns [`PagedbError::AppendNotMonotonic`] if `key` is not + /// strictly greater than the previously-appended key in this txn. + pub async fn put_append(&mut self, key: &[u8], value: &[u8]) -> Result<()> { + self.btree.put_append(key, value).await + } + + pub async fn delete(&mut self, key: &[u8]) -> Result { + self.btree.delete(key).await + } + + pub async fn put_batch(&mut self, sorted: Vec<(Vec, Vec)>) -> Result<()> { + self.btree.put_batch(sorted).await + } + + pub async fn delete_batch(&mut self, sorted: Vec>) -> Result<()> { + self.btree.delete_batch(sorted).await + } + + pub async fn delete_range(&mut self, start: &[u8], end: &[u8]) -> Result { + self.btree.delete_range(start, end).await + } + + /// Return a `CounterRef` scoped to this transaction for the named counter. + /// + /// The returned handle borrows `self` mutably for its lifetime. Use the + /// counter, drop the handle, then continue with other transaction + /// operations. The name must be at most `MAX_SEGMENT_NAME_LEN` bytes + /// (`PagedbError::NameTooLong` otherwise). + pub fn counter<'tx>(&'tx mut self, name: &str) -> Result> { + let key = Catalog::counter_key(name.as_bytes())?; + Ok(CounterRef { + catalog_tree: &mut self.catalog_tree, + main_tree: &mut self.btree, + key, + }) + } + + /// Register a segment under `name` in the catalog and schedule promotion + /// of its staging file to the live path on commit. + pub async fn link_segment(&mut self, name: &str, meta: &SegmentMeta) -> Result<()> { + let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; + if self.catalog_tree.get(&key).await?.is_some() { + return Err(PagedbError::AlreadyLinked); + } + self.enforce_segment_bytes_quota(meta.realm_id, meta.total_bytes, 0) + .await?; + let value = Catalog::encode_segment_meta(meta); + self.sync_allocator_to_catalog(); + self.catalog_tree.put(&key, &value).await?; + self.sync_allocator_from_catalog(); + self.pending_segments.push(SegmentSideEffect::Promote { + segment_id: meta.segment_id, + }); + Ok(()) + } + + /// Remove the catalog row for `name` and schedule a tombstone rename on commit. + pub async fn unlink_segment(&mut self, name: &str) -> Result<()> { + let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; + let value = self + .catalog_tree + .get(&key) + .await? + .ok_or(PagedbError::NotLinked)?; + let meta = Catalog::decode_segment_meta(&value)?; + self.sync_allocator_to_catalog(); + let removed = self.catalog_tree.delete(&key).await?; + self.sync_allocator_from_catalog(); + if !removed { + return Err(PagedbError::NotLinked); + } + self.pending_segments.push(SegmentSideEffect::Tombstone { + segment_id: meta.segment_id, + }); + Ok(()) + } + + /// Atomically swap the segment recorded under `name`: tombstone the old + /// segment id and promote `new_meta`'s staging file on commit. + pub async fn replace_segment(&mut self, name: &str, new_meta: &SegmentMeta) -> Result<()> { + let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; + let existing = self + .catalog_tree + .get(&key) + .await? + .ok_or(PagedbError::NotLinked)?; + let old_meta = Catalog::decode_segment_meta(&existing)?; + self.enforce_segment_bytes_quota( + new_meta.realm_id, + new_meta.total_bytes, + old_meta.total_bytes, + ) + .await?; + let value = Catalog::encode_segment_meta(new_meta); + self.sync_allocator_to_catalog(); + self.catalog_tree.put(&key, &value).await?; + self.sync_allocator_from_catalog(); + self.pending_segments.push(SegmentSideEffect::Tombstone { + segment_id: old_meta.segment_id, + }); + self.pending_segments.push(SegmentSideEffect::Promote { + segment_id: new_meta.segment_id, + }); + Ok(()) + } + + /// Check if the realm's committed segment bytes plus `new_bytes` minus + /// `delta_remove_bytes` would exceed the configured cap. Returns `Ok(())` + /// when no cap is set or the projected total is within the limit. + async fn enforce_segment_bytes_quota( + &self, + realm: RealmId, + new_bytes: u64, + delta_remove_bytes: u64, + ) -> Result<()> { + let quota_key = Catalog::quota_key(realm); + let quotas = match self.catalog_tree.get("a_key).await? { + Some(v) => Catalog::decode_realm_quotas(&v)?, + None => RealmQuotas::default(), + }; + let Some(limit) = quotas.max_segment_bytes else { + return Ok(()); + }; + // Scan all catalog segment rows for this realm to sum committed bytes. + let mut prefix = Vec::with_capacity(17); + prefix.push(0x01u8); // CatalogRowKind::Segment + prefix.extend_from_slice(&realm.0); + let mut end = prefix.clone(); + end.push(0xFF); + let rows = self.catalog_tree.collect_range(&prefix, &end).await?; + let mut committed: u64 = 0; + for (_, v) in rows { + let meta = Catalog::decode_segment_meta(&v)?; + committed = committed.saturating_add(meta.total_bytes); + } + let after_remove = committed.saturating_sub(delta_remove_bytes); + let projected = after_remove.saturating_add(new_bytes); + if projected > limit { + return Err(PagedbError::quota( + realm, + QuotaKind::SegmentBytes, + projected, + limit, + )); + } + Ok(()) + } + + /// Discard all in-flight dirty pages without writing anything durable. + /// The spill tmp file (if created) is removed before returning (best-effort; + /// errors are ignored). + pub async fn abort(mut self) { + tracing::debug!(name = "txn.abort", "write transaction aborted"); + self.db.pager.discard_dirty_main(self.db.realm_id); + self.committed_or_aborted = true; + self.db + .spill_bytes_in_use + .store(0, std::sync::atomic::Ordering::Relaxed); + self.cleanup_spill_async().await; + } + + /// Before a catalog operation: ensure the catalog tree's allocator cursor + /// is at least as high as the main tree's, so neither tree allocates the + /// same page id. + pub(super) fn sync_allocator_to_catalog(&mut self) { + let main_next = self.btree.next_page_id(); + let cat_next = self.catalog_tree.next_page_id(); + let shared = main_next.max(cat_next); + self.catalog_tree.set_next_page_id(shared); + } + + /// After a catalog operation: propagate any catalog allocation advances + /// back to the main tree so subsequent main-tree allocations stay above. + pub(super) fn sync_allocator_from_catalog(&mut self) { + let cat_next = self.catalog_tree.next_page_id(); + self.btree.set_next_page_id(cat_next); + } +} + +impl Drop for WriteTxn<'_, V> { + fn drop(&mut self) { + if !self.committed_or_aborted { + self.db.pager.discard_dirty_main(self.db.realm_id); + } + } +} From e0e4da01f46f4b6d52c3c65043b2987359f91d12 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 15 Jun 2026 05:33:45 +0800 Subject: [PATCH 4/8] test: add coverage for durable free-list, compaction consistency, and stall policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deferred_free_reclaim (new): - `sustained_overwrite_commits_drain_deferred_free` — rewrites a constant key set across 200 commits with no reader pinned; asserts the free-list backlog stays bounded (pages drain to the allocator on commit rather than accumulating). - `sustained_overwrite_does_not_grow_file` — same workload, asserts the file's page high-water mark stays bounded (freed pages are recycled, not orphaned). - `bounded_history_reuses_pages_after_window` — verifies reclamation under a capped commit-history window: pages freed below the oldest retained history root are recycled once the window slides past them. compaction_incremental: - `compact_step_reopen_history_consistent` — verifies that after an incremental compaction with a dense repack, the A/B header's commit-history root is reset to 0 (the repack discards it); a subsequent write and read on the reopened store must succeed. - `compact_step_preserves_free_list` — verifies that an intermediate `compact_step` (budget too small for a full repack) preserves the durable free-list chain in the header so pages are not lost. fsck_deep: - `free_list_pages_are_accounted_not_orphans` — after populating the durable free-list by deleting keys, `run_deep_walk` must report no page issues and no orphans (chain pages and tracked free pages are correctly accounted as reachable). reader_stall_policy: - `reopen_with_drainable_backlog_does_not_brick_readers` — reproduces a regression where a store reopened with a large inherited deferred-free backlog tripped `AbortOldest` on the first post-open commit, permanently bricking readers. The backlog's entries are all older than the fresh reader's pin, so the commit must drain them and evaluate the stall policy against the post-drain depth. benches/comparison: remove the now-deleted `with_skip_freelist_persistence_when_no_readers` call; the comment is updated to explain why write latency is comparable to redb without that flag. --- benches/comparison.rs | 9 +- tests/compaction_incremental.rs | 108 ++++++++++++++++ tests/deferred_free_reclaim.rs | 223 ++++++++++++++++++++++++++++++++ tests/fsck_deep.rs | 48 +++++++ tests/reader_stall_policy.rs | 98 ++++++++++++++ 5 files changed, 481 insertions(+), 5 deletions(-) create mode 100644 tests/deferred_free_reclaim.rs diff --git a/benches/comparison.rs b/benches/comparison.rs index 13a22bc..e98e1e4 100644 --- a/benches/comparison.rs +++ b/benches/comparison.rs @@ -84,12 +84,11 @@ fn open_pagedb_fresh() -> PagedbBench { // Fair-comparison: redb has no equivalent commit-history index; // disable pagedb's so the bench measures the same feature surface. // (`Disabled` is a pagedb extension; not in the architecture spec.) + // With history disabled and no readers, the commit path recycles freed + // pages through the in-memory allocator cache without persisting a + // deferred-free row, so per-commit work matches redb's for a fair + // write-latency comparison. .with_commit_history_retain(RetainPolicy::Disabled) - // Fair-comparison: redb does not persist a deferred-free queue per - // commit. Opt into pagedb's fast-free path so write-latency numbers - // measure the same per-commit work. Production embedders that turn - // this on must compact periodically. - .with_skip_freelist_persistence_when_no_readers(true) .with_metrics_enabled(false) // Large bulk loads need a generous nonce budget per txn. .with_anchor_budget(10_000_000); diff --git a/tests/compaction_incremental.rs b/tests/compaction_incremental.rs index 8786a69..83d1d62 100644 --- a/tests/compaction_incremental.rs +++ b/tests/compaction_incremental.rs @@ -222,3 +222,111 @@ async fn compact_now_completes_fully() { ); } } + +// ─── Test: history is consistently discarded across compact_step (no dangling +// commit-history root in the durable header) ──────────────────────────── +#[tokio::test(flavor = "current_thread")] +async fn compact_step_reopen_history_consistent() { + let vfs = MemVfs::new(); + { + // Default options retain commit history (Count(1024)). Build several + // commits so a history tree exists, then incrementally compact through + // intermediate steps and a final dense repack. + let db = Db::open_internal(vfs.clone(), KEK, PAGE, REALM) + .await + .unwrap(); + for round in 0u32..15 { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..30 { + w.put(format!("k{round:02}_{i:04}").as_bytes(), &[round as u8; 64]) + .await + .unwrap(); + } + w.commit().await.unwrap(); + } + let budget = CompactBudget::new(5, 10_000); // small → intermediate steps + final + let mut iters = 0; + loop { + let p = db.compact_step(budget).await.unwrap(); + iters += 1; + assert!(iters < 10_000, "compaction did not converge"); + if !p.more_work { + break; + } + } + // Db drops here. + } + + // Reopen. Before the fix, the header still pointed at the pre-repack + // commit-history root (overwritten/truncated by the dense repack), so the + // next write — which opens the history tree at that root — corrupted or + // errored. It must now be a clean reset (root = 0). + let db = Db::open_existing(vfs.clone(), KEK, PAGE, REALM) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + w.put(b"after-reopen", b"v").await.unwrap(); + w.commit().await.unwrap(); + } + let r = db.begin_read().await.unwrap(); + assert_eq!( + r.get(b"after-reopen").await.unwrap().as_deref(), + Some(&b"v"[..]) + ); + // Pre-compaction data survived the repack. + assert!( + r.get(b"k00_0000").await.unwrap().is_some(), + "data written before compaction must survive" + ); +} + +// ─── Test: an intermediate compact_step preserves the durable free-list ────── +#[tokio::test(flavor = "current_thread")] +async fn compact_step_preserves_free_list() { + // No commit history, so freed pages are immediately reclaimable and tracked + // in the durable free-list with no reader/history pinning them. + let opts = pagedb::options::OpenOptions::default() + .with_commit_history_retain(pagedb::options::RetainPolicy::Disabled); + let db = Db::open_internal_with_options(MemVfs::new(), KEK, PAGE, REALM, opts) + .await + .unwrap(); + + // Build a working set, then delete most of it to populate the free-list. + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..300 { + w.put(format!("k{i:05}").as_bytes(), &[1u8; 128]) + .await + .unwrap(); + } + w.commit().await.unwrap(); + } + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..250 { + w.delete(format!("k{i:05}").as_bytes()).await.unwrap(); + } + w.commit().await.unwrap(); + } + let pending_before = db.stats().await.unwrap().free_list_pending_entries; + assert!( + pending_before > 0, + "setup should have populated the free-list; got {pending_before}" + ); + + // One intermediate step (small budget so it is NOT the final batch). + let prog = db + .compact_step(CompactBudget::new(5, 10_000)) + .await + .unwrap(); + assert!(prog.more_work, "small budget should leave more work"); + + // The pre-existing free-list must survive the intermediate step, not be + // wiped — those pages are still reusable by ordinary writes. + let pending_after = db.stats().await.unwrap().free_list_pending_entries; + assert!( + pending_after >= pending_before, + "intermediate compact_step wiped the durable free-list: {pending_before} -> {pending_after}" + ); +} diff --git a/tests/deferred_free_reclaim.rs b/tests/deferred_free_reclaim.rs new file mode 100644 index 0000000..5ab6247 --- /dev/null +++ b/tests/deferred_free_reclaim.rs @@ -0,0 +1,223 @@ +//! Deferred-free reclamation on the commit path. +//! +//! When no reader pins an old snapshot, pages freed by a commit are eligible +//! for immediate reuse. A store that rewrites a bounded working set over many +//! one-commit-per-write transactions must therefore stay bounded in both +//! file size and deferred-free backlog depth — the freed pages have to flow +//! back to the allocator instead of accumulating in the deferred-free queue. +//! +//! These tests pin the *bounded* contract: they rewrite a constant key set +//! across a growing number of commits and assert that neither the file's +//! high-water mark nor the deferred-free backlog grows in proportion to the +//! commit count. + +use pagedb::options::{OpenOptions, RetainPolicy}; +use pagedb::vfs::memory::MemVfs; +use pagedb::{Db, RealmId}; + +const KEK: [u8; 32] = [0x2Au8; 32]; +const REALM: RealmId = RealmId::new([0x5Cu8; 16]); +const PAGE: usize = 4096; + +/// A constant working set: the same keys rewritten every round so that the +/// live data size never changes — only the commit count does. +const LIVE_KEYS: u32 = 20; +const VALUE: [u8; 256] = [0xABu8; 256]; + +/// These tests isolate free-page *reuse*, so commit history is disabled: a +/// retained historical root legitimately pins the pages of every prior commit +/// (that's the point of retention), which would mask reclamation. Reclamation +/// under a *bounded* history window is covered separately by +/// [`bounded_history_reuses_pages_after_window`]. +fn no_history_opts() -> OpenOptions { + OpenOptions::default().with_commit_history_retain(RetainPolicy::Disabled) +} + +async fn fresh_db() -> Db { + Db::open_internal_with_options(MemVfs::new(), KEK, PAGE, REALM, no_history_opts()) + .await + .unwrap() +} + +/// Rewrite the whole working set in one transaction, one commit per round. +async fn overwrite_round(db: &Db, round: u32) { + let mut w = db.begin_write().await.unwrap(); + for i in 0..LIVE_KEYS { + let mut v = VALUE; + v[0] = (round % 251) as u8; + w.put(format!("k{i:04}").as_bytes(), &v).await.unwrap(); + } + w.commit().await.unwrap(); +} + +/// With no readers pinning, the deferred-free backlog must not accumulate in +/// proportion to the number of commits: eligible freed pages drain back to the +/// allocator on commit. A constant working set rewritten N times should leave +/// only a bounded backlog, not one that scales with N. +#[tokio::test(flavor = "current_thread")] +async fn sustained_overwrite_commits_drain_deferred_free() { + let db = fresh_db().await; + + // Seed the working set. + overwrite_round(&db, 0).await; + + // No reader is ever opened, so every freed page is immediately eligible + // for reuse. Run enough rounds that an un-drained queue would be obvious. + for round in 1..=200 { + overwrite_round(&db, round).await; + } + + let stats = db.stats().await.unwrap(); + assert!( + stats.free_list_pending_entries < 64, + "deferred-free backlog must stay bounded when no readers pin — it must \ + drain on commit, not accumulate one entry per freed page; after 200 \ + commits of a {LIVE_KEYS}-key working set the backlog was {} entries", + stats.free_list_pending_entries, + ); +} + +/// Rewriting a constant working set must reuse freed pages, so the file's +/// high-water mark (`next_page_id`) stays bounded regardless of how many +/// commits were issued. Today the allocator only bump-allocates across +/// commits, so `next_page_id` grows linearly with the commit count. +#[tokio::test(flavor = "current_thread")] +async fn sustained_overwrite_commits_keep_file_bounded() { + let db = fresh_db().await; + + overwrite_round(&db, 0).await; + + for round in 1..=200 { + overwrite_round(&db, round).await; + } + + let stats = db.stats().await.unwrap(); + // The working set is ~20 keys × 256 B; the entire live tree fits in a + // double-digit page count. The bound here is a constant — it must NOT + // scale with the 200 commits issued. + assert!( + stats.main_db_next_page_id < 200, + "file high-water mark must stay bounded for a constant working set; \ + after 200 one-commit-per-write rounds next_page_id was {} (linear \ + growth means freed pages were never reused)", + stats.main_db_next_page_id, + ); +} + +/// The bound must hold independent of commit count: doubling the number of +/// commits over the same working set must not roughly double the file size. +/// This is the direct guard against the super-linear growth reported. +#[tokio::test(flavor = "current_thread")] +async fn file_growth_is_independent_of_commit_count() { + let measure = |rounds: u32| async move { + let db = fresh_db().await; + overwrite_round(&db, 0).await; + for round in 1..=rounds { + overwrite_round(&db, round).await; + } + db.stats().await.unwrap().main_db_next_page_id + }; + + let at_100 = measure(100).await; + let at_400 = measure(400).await; + + // 4× the commits over the same data must not meaningfully grow the file. + assert!( + at_400 <= at_100 + 16, + "file size scales with commit count (super-linear growth): \ + next_page_id was {at_100} after 100 commits and {at_400} after 400 \ + commits of the same {LIVE_KEYS}-key working set", + ); +} + +/// With a *bounded* commit-history window, pages reachable only from commits +/// older than the window become reclaimable once those commits are pruned, so +/// sustained writes stay bounded a constant factor above the window size — +/// they do not grow without bound the way the unbounded deferred-free queue +/// did. This is the reported default-configuration daemon scenario. +#[tokio::test(flavor = "current_thread")] +async fn bounded_history_reuses_pages_after_window() { + const WINDOW: u32 = 8; + let opts = OpenOptions::default().with_commit_history_retain(RetainPolicy::Count(WINDOW)); + let db = Db::open_internal_with_options(MemVfs::new(), KEK, PAGE, REALM, opts) + .await + .unwrap(); + + overwrite_round(&db, 0).await; + for round in 1..=300 { + overwrite_round(&db, round).await; + } + + let stats = db.stats().await.unwrap(); + // Live working set + a bounded history window of CoW deltas + the bounded + // deferred-free queue that holds the window's not-yet-prunable frees. This + // is a small constant — emphatically not the ~900 pages 300 unreclaimed + // commits would produce. + assert!( + stats.main_db_next_page_id < 200, + "bounded history must keep the file bounded; after 300 commits with a \ + {WINDOW}-commit window next_page_id was {}", + stats.main_db_next_page_id, + ); +} + +/// The free list is durable: pages freed before an *unclean* shutdown (no +/// compaction, no graceful drain) are recovered on reopen and recycled, so the +/// file does not grow when the freed space is re-used after a restart. This is +/// the crash-durability guarantee — under an in-memory-only free pool those +/// pages would be orphaned until a compaction. +#[tokio::test(flavor = "current_thread")] +async fn free_list_survives_unclean_reopen() { + let vfs = MemVfs::new(); + + // Write a working set, then delete it — freeing many pages into the durable + // free list — and drop the handle without compacting or draining. + let high_water_before; + { + let db = Db::open_internal_with_options(vfs.clone(), KEK, PAGE, REALM, no_history_opts()) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..400 { + w.put(format!("k{i:05}").as_bytes(), &VALUE).await.unwrap(); + } + w.commit().await.unwrap(); + } + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..400 { + w.delete(format!("k{i:05}").as_bytes()).await.unwrap(); + } + w.commit().await.unwrap(); + } + high_water_before = db.stats().await.unwrap().main_db_next_page_id; + let pending = db.stats().await.unwrap().free_list_pending_entries; + assert!( + pending > 20, + "deletes should have freed pages into the durable free list; got {pending}" + ); + // Drop without compaction or graceful shutdown — simulates a crash. + } + + // Reopen and write a fresh working set. With the free list recovered from + // disk, these allocations recycle the freed pages instead of extending the + // file past where it already was. + let db = Db::open_existing_with_options(vfs.clone(), KEK, PAGE, REALM, no_history_opts()) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..400 { + w.put(format!("n{i:05}").as_bytes(), &VALUE).await.unwrap(); + } + w.commit().await.unwrap(); + } + let high_water_after = db.stats().await.unwrap().main_db_next_page_id; + assert!( + high_water_after <= high_water_before + 8, + "freed pages must be recovered from the durable free list and recycled \ + after an unclean reopen; high-water was {high_water_before} before the \ + crash and {high_water_after} after rewriting the same volume" + ); +} diff --git a/tests/fsck_deep.rs b/tests/fsck_deep.rs index ded2309..96d9f5f 100644 --- a/tests/fsck_deep.rs +++ b/tests/fsck_deep.rs @@ -102,3 +102,51 @@ async fn corrupt_page_detected() { ); } } + +#[tokio::test(flavor = "current_thread")] +async fn free_list_pages_are_accounted_not_orphans() { + // Disable history so deleted pages enter the durable free-list immediately + // (no retained snapshot pins them). + let opts = OpenOptions::default() + .with_buffer_pool_pages(64) + .with_commit_history_retain(pagedb::options::RetainPolicy::Disabled); + let db = Db::open_internal_with_options(MemVfs::new(), KEK, 4096, REALM, opts) + .await + .unwrap(); + + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..300 { + w.put(format!("k{i:05}").as_bytes(), &[7u8; 128]) + .await + .unwrap(); + } + w.commit().await.unwrap(); + } + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..250 { + w.delete(format!("k{i:05}").as_bytes()).await.unwrap(); + } + w.commit().await.unwrap(); + } + assert!( + db.stats().await.unwrap().free_list_pending_entries > 0, + "setup should have populated the durable free-list" + ); + + let report = run_deep_walk(&db).await.unwrap(); + assert!( + report.page_issues.is_empty(), + "free-listed/chain pages must verify cleanly, got: {:?}", + report.page_issues + ); + // The free-list chain pages and the pages they track are accounted for, so + // none should be reported as orphans. + assert!( + report.orphan_page_ids.is_empty(), + "free-list pages must not be reported as orphans, got: {:?}", + report.orphan_page_ids + ); + assert!(report.is_clean(), "report should be clean"); +} diff --git a/tests/reader_stall_policy.rs b/tests/reader_stall_policy.rs index a01f0ec..8998883 100644 --- a/tests/reader_stall_policy.rs +++ b/tests/reader_stall_policy.rs @@ -124,6 +124,104 @@ async fn abort_oldest_aborts_old_reader() { drop(r2); } +/// After an unclean shutdown that leaves a *drainable* deferred-free backlog +/// behind, reopening the store must self-heal: the inherited backlog is not +/// corruption and must not brick the store. The deferred entries were tagged +/// with commit ids older than anything a freshly-opened reader pins, so the +/// first post-open commit drains them — the stall policy is evaluated against +/// the post-drain depth, so a reader opened against the reopened store is +/// neither rejected nor aborted. +/// +/// Repro of the reported failure: a daemon under write load is killed with a +/// deferred-free backlog past the stall threshold; on restart the consumer +/// holds a read txn across its schema-ensure write, and that first commit trips +/// `AbortOldest` against the backlog inherited from the dead writer — surfacing +/// as a permanent `Aborted` on every reopen. +#[tokio::test(flavor = "current_thread")] +async fn reopen_with_drainable_backlog_does_not_brick_readers() { + const THRESHOLD: u64 = 50; + let opts = || { + let mut o = OpenOptions::default() + .with_buffer_pool_pages(256) + // Disable commit history so the backlog is governed purely by the + // reader-pin floor: once the dead writer's reader is gone, every + // inherited entry is drainable. (Reclamation under a *bounded* + // history window is covered by `deferred_free_reclaim`.) + .with_commit_history_retain(pagedb::options::RetainPolicy::Disabled); + o.reader_stall_threshold_pages = THRESHOLD; + o + }; + let vfs = MemVfs::new(); + + // Phase 1: build a large *durable* deferred-free backlog. Holding a reader + // pins an old commit, forcing every subsequent commit's freed pages into + // the deferred-free queue (none are eligible to drain while that reader + // pins). Then drop everything without draining — simulating a writer killed + // mid-workload. The backlog persists on disk in the catalog. + { + let db = Db::open(vfs.clone(), KEK, 4096, REALM, opts()) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + w.put(b"seed", b"v").await.unwrap(); + w.commit().await.unwrap(); + } + // Pin an old snapshot for the duration of the build. AbortOldest may + // flag it, but it keeps pinning until dropped, so the backlog grows. + let pin = db.begin_read().await.unwrap(); + for round in 0u32..40 { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..8 { + w.put(format!("k{round:03}_{i}").as_bytes(), &[0u8; 128]) + .await + .unwrap(); + } + w.commit().await.unwrap(); + let mut w2 = db.begin_write().await.unwrap(); + for i in 0u32..8 { + w2.put(format!("k{round:03}_{i}").as_bytes(), b"x") + .await + .unwrap(); + } + w2.commit().await.unwrap(); + } + let pending = db.stats().await.unwrap().free_list_pending_entries; + assert!( + pending > THRESHOLD, + "test setup should have built a backlog past the threshold; got {pending}" + ); + drop(pin); + // Handle drops here — backlog is durably on disk, undrained. + } + + // Phase 2: reopen (process restart) and run the consumer open pattern: a + // read txn held across a single write commit (schema ensure). The inherited + // backlog's entries are all older than the reopened reader's pin, so the + // commit drains them and the stall policy sees a depth below the threshold. + let db = Db::open(vfs.clone(), KEK, 4096, REALM, opts()) + .await + .unwrap(); + let reader = db.begin_read().await.unwrap(); + let mut w = db.begin_write().await.unwrap(); + w.put(b"schema", b"v1").await.unwrap(); + let res = w.commit().await; + assert!( + res.is_ok(), + "post-reopen commit must not be rejected by the stall policy on an \ + inherited drainable backlog: {res:?}" + ); + + let got = reader.get(b"seed").await; + assert!( + !matches!(got, Err(PagedbError::Aborted)), + "reopening a store with a drainable backlog bricked a live reader with \ + Aborted — a non-corrupt, recoverable backlog must self-heal" + ); + assert!(got.is_ok(), "reader read failed after reopen: {got:?}"); + drop(reader); +} + #[tokio::test(flavor = "current_thread")] async fn non_abortable_reader_survives_abort_oldest() { let vfs = MemVfs::new(); From 9de0305c5baca9b3b17b6f6804d1044fae0edadd Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 15 Jun 2026 06:07:47 +0800 Subject: [PATCH 5/8] fix(ci): keep rocksdb out of the test build; install libclang for bench compiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bench targets now carry `test = false`, which prevents `cargo test` and nextest from compiling `harness = false` benches and running their `main` as test cases. Without that flag, building the comparison bench dragged in rocksdb → librocksdb-sys → bindgen → clang-sys → libclang on every platform and also executed the benchmark binary during the test run. The two Ubuntu jobs that explicitly compile benches (`clippy --all-targets` and `cargo check --benches`) still need libclang, so both now install clang and libclang-dev before the Rust cache step. --- .github/workflows/test.yml | 10 ++++++++++ Cargo.toml | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5823a4..61ba4c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,6 +46,11 @@ jobs: with: components: rustfmt, clippy + # The comparison benchmark links rocksdb (librocksdb-sys), whose build + # runs bindgen and needs libclang. clippy --all-targets compiles it. + - name: Install LLVM/Clang + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: Swatinem/rust-cache@v2 with: prefix-key: lint @@ -198,6 +203,11 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + # --benches compiles the comparison bench, which links rocksdb + # (librocksdb-sys) — its build runs bindgen and needs libclang. + - name: Install LLVM/Clang + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: Swatinem/rust-cache@v2 with: prefix-key: features diff --git a/Cargo.toml b/Cargo.toml index 2b73ac1..0ca73ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,17 +129,24 @@ fastrand = "2" name = "pagedb-fsck" path = "src/bin/pagedb-fsck.rs" +# `test = false`: these are benchmarks, not tests. Without it, `cargo test` +# (and nextest) compile every `harness = false` bench and run its `main` as a +# test — pulling heavy dev-deps (rocksdb → libclang) into the test build on +# every platform and executing benchmarks during the test run. [[bench]] name = "btree" harness = false +test = false [[bench]] name = "segment" harness = false +test = false [[bench]] name = "comparison" harness = false +test = false [profile.release] lto = "thin" From dbdfd483dfb68c1e0715a3708bcdd3cfbe3e34f1 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 15 Jun 2026 06:07:55 +0800 Subject: [PATCH 6/8] fix(vfs): treat directory-open failure as a no-op in sync_dir on Windows On Windows, opening a directory as a file handle fails with PermissionDenied ("Access is denied") before sync_all is ever reached. sync_dir's existing logic already treated PermissionDenied and Unsupported from sync_all as best-effort no-ops (directory fsync is advisory there), but the open failure propagated as an error instead. Extend the same no-op treatment to the open call itself so that SegmentWriter::create, which calls sync_dir on "seg/.staging", does not brick on Windows. --- src/vfs/tokio_backend.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/vfs/tokio_backend.rs b/src/vfs/tokio_backend.rs index 80cf16c..4bf8644 100644 --- a/src/vfs/tokio_backend.rs +++ b/src/vfs/tokio_backend.rs @@ -419,6 +419,18 @@ impl Vfs for TokioVfs { let dir = match std::fs::File::open(&p) { Ok(d) => d, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + // Opening a directory as a file handle is unsupported on some + // platforms — notably Windows, where it fails with PermissionDenied + // ("Access is denied") at open, before sync_all is ever reached. + // Directory fsync is best-effort there, so treat it as a no-op. + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::Unsupported | std::io::ErrorKind::PermissionDenied + ) => + { + return Ok(()); + } Err(e) => return Err(PagedbError::Io(e)), }; match dir.sync_all() { From 874d51d1f8021d73e9bc802e0c6d50c017e6f49c Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 15 Jun 2026 06:25:04 +0800 Subject: [PATCH 7/8] fix(vfs): keep GCD backend read/write futures Send MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dispatch_io handler RcBlock and the raw block pointer derived from it are !Send. When they were constructed inline inside read_at/write_at, they lived in the async fn's state machine across the oneshot .await, making the returned futures !Send — breaking cargo check on every Apple target because VfsFile requires Send futures. Extracted the block construction and channel submission into synchronous free functions submit_read/submit_write. The !Send values are created, used, and dropped entirely within those functions before any await point is reached, so the calling futures remain Send. Verified with: cargo check --target aarch64-apple-darwin --lib cargo check --target x86_64-apple-darwin --lib cargo check --target aarch64-apple-ios --lib --- src/vfs/gcd/file.rs | 211 +++++++++++++++++++++++++------------------- 1 file changed, 119 insertions(+), 92 deletions(-) diff --git a/src/vfs/gcd/file.rs b/src/vfs/gcd/file.rs index c6fdae1..78ca888 100644 --- a/src/vfs/gcd/file.rs +++ b/src/vfs/gcd/file.rs @@ -92,6 +92,117 @@ impl Drop for GcdFile { } } +/// Submit a `dispatch_io` read and return immediately. Kept synchronous on +/// purpose: the handler `RcBlock` and the raw block pointer derived from it are +/// `!Send`, so they must never be live across an await point in the calling +/// future (`VfsFile::read_at` must return a `Send` future). The handler +/// accumulates chunks and, on completion, sends the result through `tx`. +fn submit_read( + channel: &DispatchIO, + queue: &DispatchQueue, + offset: u64, + len: usize, + tx: tokio::sync::oneshot::Sender>>, +) { + let accum: Arc>> = Arc::new(Mutex::new(Vec::with_capacity(len))); + let sender = Arc::new(Mutex::new(Some(tx))); + + // block2 cannot encode `bool` or `*mut DispatchData` directly (the + // dispatch_object! types don't implement `RefEncode`). We declare the + // closure with ABI-compatible stand-ins — `u8` for bool and `*mut c_void` + // for DispatchData — and transmute the resulting block pointer to the + // dispatch_io_handler_t signature. dispatch2's own `DispatchData::to_vec` + // uses the same workaround (see data.rs:114). + let handler: RcBlock = + RcBlock::new(move |done: u8, data: *mut c_void, error: libc::c_int| { + if !data.is_null() { + let d: &DispatchData = + // SAFETY: dispatch guarantees `data` (when non-null) is a + // valid `DispatchData` retained for the handler's duration. + unsafe { &*data.cast::() }; + let bytes = d.to_vec(); + accum.lock().extend_from_slice(&bytes); + } + if done != 0 { + let result = if error != 0 { + Err(std::io::Error::from_raw_os_error(error)) + } else { + Ok(std::mem::take(&mut *accum.lock())) + }; + if let Some(s) = sender.lock().take() { + let _ = s.send(result); + } + } + }); + + // SAFETY: transmute is from the stand-in block signature to the typedef + // declared by dispatch2 — the ABI is identical because `bool` and `u8` + // share the same one-byte ABI, and a `*mut DispatchData` is bit-identical + // to a `*mut c_void`. + let handler_ptr: *mut DynBlock = unsafe { + std::mem::transmute::< + *mut Block, + *mut DynBlock, + >(RcBlock::as_ptr(&handler)) + }; + + // SAFETY: channel and queue are valid (owned by the caller's `GcdFile`); + // the handler block is retained by libdispatch on submission. + unsafe { + #[allow(clippy::cast_possible_wrap)] + channel.read(offset as libc::off_t, len, queue, handler_ptr); + } + // libdispatch has retained the block internally; we can drop our Rc. + drop(handler); +} + +/// Submit a `dispatch_io` write and return immediately. Synchronous for the +/// same reason as [`submit_read`]: the handler block, its raw pointer, and the +/// `DispatchData` are all `!Send` and must not cross an await point. +fn submit_write( + channel: &DispatchIO, + queue: &DispatchQueue, + offset: u64, + buf: &[u8], + tx: tokio::sync::oneshot::Sender>, +) { + let data = DispatchData::from_bytes(buf); + let sender = Arc::new(Mutex::new(Some(tx))); + + // Same bool→u8 / *mut DispatchData→*mut c_void workaround as `submit_read`. + let handler: RcBlock = RcBlock::new( + move |done: u8, _remaining: *mut c_void, error: libc::c_int| { + if done != 0 { + let result = if error != 0 { + Err(std::io::Error::from_raw_os_error(error)) + } else { + Ok(()) + }; + if let Some(s) = sender.lock().take() { + let _ = s.send(result); + } + } + }, + ); + + // SAFETY: ABI-compatible transmute; see `submit_read`. + let handler_ptr: *mut DynBlock = unsafe { + std::mem::transmute::< + *mut Block, + *mut DynBlock, + >(RcBlock::as_ptr(&handler)) + }; + + // SAFETY: channel/queue/data all valid; libdispatch retains both the data + // object and the handler block for the operation's duration. + unsafe { + #[allow(clippy::cast_possible_wrap)] + channel.write(offset as libc::off_t, &data, queue, handler_ptr); + } + drop(handler); + drop(data); +} + impl VfsFile for GcdFile { async fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result { if buf.is_empty() { @@ -99,60 +210,12 @@ impl VfsFile for GcdFile { } let len = buf.len(); let (tx, rx) = tokio::sync::oneshot::channel::>>(); - let accum: Arc>> = Arc::new(Mutex::new(Vec::with_capacity(len))); - let sender = Arc::new(Mutex::new(Some(tx))); - - // block2 cannot encode `bool` or `*mut DispatchData` directly (the - // dispatch_object! types don't implement `RefEncode`). We declare the - // closure with ABI-compatible stand-ins — `u8` for bool and - // `*mut c_void` for DispatchData — and transmute the resulting block - // pointer to the dispatch_io_handler_t signature. dispatch2's own - // `DispatchData::to_vec` uses the same workaround (see data.rs:114). - let accum_for_block = Arc::clone(&accum); - let sender_for_block = Arc::clone(&sender); - let handler: RcBlock = - RcBlock::new(move |done: u8, data: *mut c_void, error: libc::c_int| { - if !data.is_null() { - let d: &DispatchData = - // SAFETY: dispatch guarantees `data` (when non-null) - // is a valid `DispatchData` retained for the - // handler's duration. - unsafe { &*data.cast::() }; - let bytes = d.to_vec(); - accum_for_block.lock().extend_from_slice(&bytes); - } - if done != 0 { - let result = if error != 0 { - Err(std::io::Error::from_raw_os_error(error)) - } else { - Ok(std::mem::take(&mut *accum_for_block.lock())) - }; - if let Some(s) = sender_for_block.lock().take() { - let _ = s.send(result); - } - } - }); - - // SAFETY: transmute is from the stand-in block signature to the - // typedef declared by dispatch2 — the ABI is identical because `bool` - // and `u8` share the same one-byte ABI, and a `*mut DispatchData` is - // bit-identical to a `*mut c_void`. - let handler_ptr: *mut DynBlock = unsafe { - std::mem::transmute::< - *mut Block, - *mut DynBlock, - >(RcBlock::as_ptr(&handler)) - }; - - // SAFETY: the channel and queue are valid (held by `self`); the - // handler block is retained by libdispatch on submission. - unsafe { - #[allow(clippy::cast_possible_wrap)] - self.channel - .read(offset as libc::off_t, len, &self.queue, handler_ptr); - } - // libdispatch has retained the block internally; we can drop our Rc. - drop(handler); + // The handler block and its raw block pointer are `!Send`; submitting + // from a synchronous helper keeps them out of this future's state + // machine, so the future stays `Send` as `VfsFile` requires. Once + // submitted, libdispatch owns the operation and reports back through the + // oneshot, which is all we await here. + submit_read(&self.channel, &self.queue, offset, len, tx); let data = rx .await @@ -184,45 +247,9 @@ impl VfsFile for GcdFile { return Ok(0); } let len = buf.len(); - let data = DispatchData::from_bytes(buf); let (tx, rx) = tokio::sync::oneshot::channel::>(); - let sender = Arc::new(Mutex::new(Some(tx))); - - // Same bool→u8 / *mut DispatchData→*mut c_void workaround as - // `read_at`. See comments there. - let sender_for_block = Arc::clone(&sender); - let handler: RcBlock = RcBlock::new( - move |done: u8, _remaining: *mut c_void, error: libc::c_int| { - if done != 0 { - let result = if error != 0 { - Err(std::io::Error::from_raw_os_error(error)) - } else { - Ok(()) - }; - if let Some(s) = sender_for_block.lock().take() { - let _ = s.send(result); - } - } - }, - ); - - // SAFETY: ABI-compatible transmute; see `read_at`. - let handler_ptr: *mut DynBlock = unsafe { - std::mem::transmute::< - *mut Block, - *mut DynBlock, - >(RcBlock::as_ptr(&handler)) - }; - - // SAFETY: channel/queue/data all valid; libdispatch retains both the - // data object and the handler block for the operation's duration. - unsafe { - #[allow(clippy::cast_possible_wrap)] - self.channel - .write(offset as libc::off_t, &data, &self.queue, handler_ptr); - } - drop(handler); - drop(data); + // `!Send` handler/data confined to a synchronous helper; see `read_at`. + submit_write(&self.channel, &self.queue, offset, buf, tx); rx.await .map_err(|_| PagedbError::Io(std::io::Error::other("dispatch_io write cancelled")))? From ee6bc24473b872d730ac58482605d39491283af2 Mon Sep 17 00:00:00 2001 From: Farhan Syah Date: Mon, 15 Jun 2026 06:25:10 +0800 Subject: [PATCH 8/8] fix(ci): provide libclang for the test and bench jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nextest (cargo test --no-run) and cargo bench both compile the dev-dependency graph which pulls in rocksdb → librocksdb-sys → bindgen → libclang. Windows runner images already ship LLVM; Linux and macOS do not. Added an LLVM/Clang install step to the test job (apt on Linux, brew + LIBCLANG_PATH on macOS) and to the bench job (apt on Linux), so bindgen can locate libclang on every platform and the jobs no longer fail at the compile step. --- .github/workflows/test.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 61ba4c3..5d2d0bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -95,6 +95,19 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + # nextest compiles the dev-dependency graph (incl. rocksdb → + # librocksdb-sys → bindgen → libclang). Windows runner images ship LLVM; + # Linux and macOS need it installed. + - name: Install LLVM/Clang (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + + - name: Install LLVM/Clang (macOS) + if: runner.os == 'macOS' + run: | + brew install llvm + echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV" + - uses: Swatinem/rust-cache@v2 with: prefix-key: test-${{ matrix.os }} @@ -225,6 +238,11 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + # cargo bench builds the dev-dependency graph (rocksdb → librocksdb-sys → + # bindgen → libclang), even when only the btree/segment benches are named. + - name: Install LLVM/Clang + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: Swatinem/rust-cache@v2 with: prefix-key: bench