diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5823a4..5d2d0bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,6 +46,11 @@ jobs: with: components: rustfmt, clippy + # The comparison benchmark links rocksdb (librocksdb-sys), whose build + # runs bindgen and needs libclang. clippy --all-targets compiles it. + - name: Install LLVM/Clang + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: Swatinem/rust-cache@v2 with: prefix-key: lint @@ -90,6 +95,19 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + # nextest compiles the dev-dependency graph (incl. rocksdb → + # librocksdb-sys → bindgen → libclang). Windows runner images ship LLVM; + # Linux and macOS need it installed. + - name: Install LLVM/Clang (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + + - name: Install LLVM/Clang (macOS) + if: runner.os == 'macOS' + run: | + brew install llvm + echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> "$GITHUB_ENV" + - uses: Swatinem/rust-cache@v2 with: prefix-key: test-${{ matrix.os }} @@ -198,6 +216,11 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + # --benches compiles the comparison bench, which links rocksdb + # (librocksdb-sys) — its build runs bindgen and needs libclang. + - name: Install LLVM/Clang + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: Swatinem/rust-cache@v2 with: prefix-key: features @@ -215,6 +238,11 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable + # cargo bench builds the dev-dependency graph (rocksdb → librocksdb-sys → + # bindgen → libclang), even when only the btree/segment benches are named. + - name: Install LLVM/Clang + run: sudo apt-get update && sudo apt-get install -y clang libclang-dev + - uses: Swatinem/rust-cache@v2 with: prefix-key: bench diff --git a/Cargo.toml b/Cargo.toml index 2b73ac1..0ca73ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,17 +129,24 @@ fastrand = "2" name = "pagedb-fsck" path = "src/bin/pagedb-fsck.rs" +# `test = false`: these are benchmarks, not tests. Without it, `cargo test` +# (and nextest) compile every `harness = false` bench and run its `main` as a +# test — pulling heavy dev-deps (rocksdb → libclang) into the test build on +# every platform and executing benchmarks during the test run. [[bench]] name = "btree" harness = false +test = false [[bench]] name = "segment" harness = false +test = false [[bench]] name = "comparison" harness = false +test = false [profile.release] lto = "thin" diff --git a/benches/comparison.rs b/benches/comparison.rs index 13a22bc..e98e1e4 100644 --- a/benches/comparison.rs +++ b/benches/comparison.rs @@ -84,12 +84,11 @@ fn open_pagedb_fresh() -> PagedbBench { // Fair-comparison: redb has no equivalent commit-history index; // disable pagedb's so the bench measures the same feature surface. // (`Disabled` is a pagedb extension; not in the architecture spec.) + // With history disabled and no readers, the commit path recycles freed + // pages through the in-memory allocator cache without persisting a + // deferred-free row, so per-commit work matches redb's for a fair + // write-latency comparison. .with_commit_history_retain(RetainPolicy::Disabled) - // Fair-comparison: redb does not persist a deferred-free queue per - // commit. Opt into pagedb's fast-free path so write-latency numbers - // measure the same per-commit work. Production embedders that turn - // this on must compact periodically. - .with_skip_freelist_persistence_when_no_readers(true) .with_metrics_enabled(false) // Large bulk loads need a generous nonce budget per txn. .with_anchor_budget(10_000_000); diff --git a/src/btree/tree/core.rs b/src/btree/tree/core.rs index 019fd96..720da36 100644 --- a/src/btree/tree/core.rs +++ b/src/btree/tree/core.rs @@ -52,14 +52,18 @@ pub struct BTree { /// parallel-AEAD flush. In-place mutation by subsequent puts targeting /// the same leaf is allowed; no further allocation needed. pub(super) fresh_leaves: HashMap, - /// Cross-commit pool of pre-vetted reusable page IDs, shared (via `Arc`) - /// across all `BTree`s opened by the same `Db`. Allocation pops from - /// here before bumping `next_page_id`, keeping the file size bounded - /// when `OpenOptions::skip_freelist_persistence_when_no_readers` orphans - /// would otherwise accumulate. `None` for callers that haven't wired a - /// shared cache (compaction, history tree) — those keep bump-only - /// allocation. + /// Cross-commit pool of reusable page IDs, shared (via `Arc`) across the + /// main, catalog, and history `BTree`s of one `Db`. Allocation pops from + /// here before bumping `next_page_id`, recycling pages freed by earlier + /// commits (once no reader or retained-history root can observe them) so + /// the file stays bounded under sustained writes. `None` for callers that + /// haven't wired a shared cache (e.g. compaction's repack trees), which + /// keep bump-only allocation. pub(super) free_page_cache: Option>>>, + /// Sink recording page ids drawn from `free_page_cache` and reused this + /// session. The commit path removes them from the durable free-list (they + /// now hold live committed data). Shared (via `Arc`) across the txn's trees. + pub(super) free_page_consumed: Option>>>, /// Last key successfully appended via [`Self::put_append`]. Used to /// enforce the monotonic-key invariant on subsequent calls and to /// invalidate the cached path when any non-append mutation (regular @@ -96,6 +100,7 @@ impl BTree { dirty_parent_paths: HashMap::new(), fresh_leaves: HashMap::new(), free_page_cache: None, + free_page_consumed: None, append_last_key: None, append_cached_path: None, } @@ -110,22 +115,18 @@ impl BTree { } /// Wire in the `Db`'s shared free-page cache. After this call, - /// `allocate_page` will pop from the shared pool before bumping - /// `next_page_id`. Pages pushed into the pool by an earlier writer - /// commit (via [`Self::push_to_shared_cache`]) become reusable here. + /// `allocate_page` pops from the shared pool before bumping `next_page_id`. + /// The pool is loaded at `begin_write` with the durable free-list's + /// floor-safe pages, so recycling from it is always snapshot-safe. pub fn set_free_page_cache(&mut self, cache: Arc>>) { self.free_page_cache = Some(cache); } - /// Push `page_ids` into the shared free-page cache, if one is wired. - /// Used by the writer commit path when the no-reader fast-free option - /// is active: instead of orphaning freed pages, hand them to the next - /// txn's allocator. - pub fn push_to_shared_cache(&self, page_ids: &[u64]) { - if let Some(cache) = &self.free_page_cache { - let mut guard = cache.lock(); - guard.extend(page_ids.iter().copied()); - } + /// Wire the shared sink that records cache pages reused this session, so the + /// commit path can remove them from the durable free-list. Set alongside + /// [`Self::set_free_page_cache`]. + pub fn set_free_page_consumed(&mut self, consumed: Arc>>) { + self.free_page_consumed = Some(consumed); } #[must_use] @@ -148,34 +149,31 @@ impl BTree { } pub(super) fn allocate_page(&mut self) -> u64 { - // Reuse a freed page only if it is at or above the reuse threshold. - // Pages below the threshold may still be live in pinned reader snapshots. + // First, recycle a page freed earlier *in this same session*, gated by + // the reuse threshold: a page below it may still be live in a pinned + // reader's snapshot, so it can't be reused until the durable free-list + // clears it (it leaves via `drain_freed` at commit instead). if self.reuse_threshold == 0 { - // No readers pinned: recycle freely. if let Some(id) = self.freed.pop() { return id; } - // Consult the cross-commit shared cache (pages handed off by - // earlier writer commits under the no-reader fast-free option). - // Only safe to draw from this when no readers are pinned — the - // cache contract is "always safe to immediately reuse." - if let Some(cache) = &self.free_page_cache { - if let Some(id) = cache.lock().pop() { - return id; + } else if let Some(pos) = self + .freed + .iter() + .rposition(|&id| id >= self.reuse_threshold) + { + return self.freed.remove(pos); + } + // Then draw from the shared cross-commit cache. It is loaded at txn + // begin with *only* free-list pages below the reclamation floor — pages + // no live reader and no retained-history root can observe — so reusing + // them is safe regardless of `reuse_threshold`. Record each draw so the + // commit path removes it from the durable free-list. + if let Some(cache) = &self.free_page_cache { + if let Some(id) = cache.lock().pop() { + if let Some(consumed) = &self.free_page_consumed { + consumed.lock().push(id); } - } - } else { - // Readers pinned: only recycle pages that were allocated during - // this session (>= reuse_threshold) and thus cannot be in any - // prior snapshot. The shared cache is bypassed here because - // pages in it predate this session and may be visible to a - // pinned reader at an older commit. - if let Some(pos) = self - .freed - .iter() - .rposition(|&id| id >= self.reuse_threshold) - { - let id = self.freed.remove(pos); return id; } } diff --git a/src/btree/tree/scan.rs b/src/btree/tree/scan.rs index 9880332..fa8b6f0 100644 --- a/src/btree/tree/scan.rs +++ b/src/btree/tree/scan.rs @@ -38,6 +38,20 @@ impl BTree { } } + /// Return the smallest key in the tree, or `None` if the tree is empty. + /// Descends the leftmost spine only — O(tree height), not O(tree size). + pub async fn first_key(&self) -> Result>> { + if self.root_page_id == 0 { + return Ok(None); + } + // The empty key sorts below every stored key, so the descent lands on + // the leftmost leaf. + let path = self.path_to_leaf_for_key(&[]).await?; + let leaf_id = *path.last().expect("non-empty path"); + let leaf = self.read_leaf(leaf_id).await?; + Ok(leaf.records.first().map(|(k, _)| k.clone())) + } + /// Reverse range scan: `start` inclusive, `end` exclusive. Returns results /// in descending key order. Collects matching records forward then reverses. pub async fn scan_rev(&self, start: &[u8], end: &[u8]) -> Result, Vec)>> { diff --git a/src/catalog/codec.rs b/src/catalog/codec.rs index 0bf0ca3..12e283b 100644 --- a/src/catalog/codec.rs +++ b/src/catalog/codec.rs @@ -23,16 +23,9 @@ pub enum CatalogRowKind { /// (singleton; no name suffix). Value is `RekeyState` encoded as 13 bytes: /// `target_mk_epoch[8] || main_db_done[1] || segments_remaining_idx[4]`. RekeyState = 0x03, - /// Persistent free-list entry. Key is `[0x04] || page_id_big_endian[8]` - /// (big-endian so lexicographic sort matches numeric order). Value is empty. - FreeList = 0x04, - /// Deferred-free queue. Singleton row; key is `[0x05]`. Value is a - /// length-prefixed list of `(commit_id[8] || page_id[8])` pairs, in - /// ascending `commit_id` order. Pages here are not yet safe to reallocate - /// because a reader snapshot at or before that `commit_id` may still observe - /// them. Pairs are promoted to the free-list once no tracked reader pins - /// a `commit_id` ≤ the pair's `commit_id`. - DeferredFree = 0x05, + // 0x04 and 0x05 are reserved: they were the in-catalog free-list and + // deferred-free queue, superseded by the durable free-list chain rooted in + // the A/B header (see `crate::pager::freelist`). Do not reuse these bytes. /// Durable reader pin. One row per active cross-process read transaction. /// Key: `[0x06] || pid_u32_be[4] || lease_id_u64_be[8]` (13 bytes). /// Value: `commit_id[8] || root_page_id[8] || catalog_root_page_id[8] || @@ -200,56 +193,6 @@ impl Catalog { vec![CatalogRowKind::RekeyState as u8] } - /// Free-list entry key: `[0x04] || page_id_be[8]`. - /// Big-endian so lexicographic order == numeric order. - #[must_use] - pub fn free_list_key(page_id: u64) -> [u8; 9] { - let mut k = [0u8; 9]; - k[0] = CatalogRowKind::FreeList as u8; - k[1..9].copy_from_slice(&page_id.to_be_bytes()); - k - } - - /// Deferred-free queue key: `[0x05]` (singleton). - #[must_use] - pub fn deferred_free_key() -> Vec { - vec![CatalogRowKind::DeferredFree as u8] - } - - /// Encode a slice of `(commit_id, page_id)` pairs. Empty slice encodes to - /// empty bytes. - #[must_use] - pub fn encode_deferred_free(pairs: &[(u64, u64)]) -> Vec { - let mut out = Vec::with_capacity(pairs.len() * 16); - for (commit_id, page_id) in pairs { - out.extend_from_slice(&commit_id.to_le_bytes()); - out.extend_from_slice(&page_id.to_le_bytes()); - } - out - } - - /// Decode a deferred-free value. Returns `Err` if length is not a multiple - /// of 16. - pub fn decode_deferred_free(bytes: &[u8]) -> Result> { - if bytes.len() % 16 != 0 { - return Err(PagedbError::corruption( - crate::errors::CorruptionDetail::HeaderUnverifiable, - )); - } - let mut out = Vec::with_capacity(bytes.len() / 16); - let mut i = 0; - while i + 16 <= bytes.len() { - let mut b8 = [0u8; 8]; - b8.copy_from_slice(&bytes[i..i + 8]); - let commit_id = u64::from_le_bytes(b8); - b8.copy_from_slice(&bytes[i + 8..i + 16]); - let page_id = u64::from_le_bytes(b8); - out.push((commit_id, page_id)); - i += 16; - } - Ok(out) - } - /// Encode a `RekeyStateRow` as 13 bytes. #[must_use] pub fn encode_rekey_state(r: &RekeyStateRow) -> [u8; REKEY_STATE_LEN] { diff --git a/src/compaction/freelist.rs b/src/compaction/freelist.rs deleted file mode 100644 index c182c36..0000000 --- a/src/compaction/freelist.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! Persistent free-list operations. -//! -//! The free-list is stored in the catalog B+ tree as individual rows keyed -//! `[0x04] || page_id_be[8]`. The deferred-free queue is a single catalog row -//! `[0x05]` containing serialised `(commit_id, page_id)` pairs. -//! -//! These helpers are used both by [`WriteTxn::commit`] (to drain freed pages -//! into the deferred queue) and by [`compact_now`] (to promote eligible -//! deferred pages into the free-list and to allocate from it). - -use std::sync::Arc; - -use crate::btree::BTree; -use crate::catalog::codec::{Catalog, CatalogRowKind}; -use crate::pager::Pager; -use crate::vfs::Vfs; -use crate::{RealmId, Result}; - -/// Pop the smallest `page_id` from the persistent free-list. Returns `None` if -/// the free-list is empty. -/// -/// Modifies `cat_tree` in-place (deletes the chosen entry). The caller is -/// responsible for flushing `cat_tree` and committing the header afterward. -pub async fn alloc_page(cat_tree: &mut BTree) -> Result> { - let start = vec![CatalogRowKind::FreeList as u8]; - let mut end = start.clone(); - end.push(0xFF); - let rows = cat_tree.collect_range(&start, &end).await?; - if let Some((key, _)) = rows.into_iter().next() { - // Key layout: [0x04] || page_id_be[8] — 9 bytes total. - if key.len() == 9 { - let mut b = [0u8; 8]; - b.copy_from_slice(&key[1..9]); - let page_id = u64::from_be_bytes(b); - cat_tree.delete(&key).await?; - return Ok(Some(page_id)); - } - } - Ok(None) -} - -/// Add `page_id` to the deferred-free queue tagged with `commit_id`. The page -/// becomes eligible for reuse once no tracked reader pins a `commit_id` ≤ this -/// `commit_id`. -/// -/// Modifies `cat_tree` in-place. Caller flushes and commits. -pub async fn free_page_deferred( - cat_tree: &mut BTree, - commit_id: u64, - page_id: u64, -) -> Result { - free_pages_deferred_batch(cat_tree, commit_id, &[page_id]).await -} - -/// Batch-add multiple `page_id`s to the deferred-free queue in a single -/// catalog put. More efficient than calling `free_page_deferred` in a loop -/// when many pages need to be freed at once. -/// -/// Modifies `cat_tree` in-place. Caller flushes and commits. Returns the -/// total pair count in the deferred-free row after the append; callers -/// (e.g. the stall-policy check) can use this to avoid re-reading the row. -pub async fn free_pages_deferred_batch( - cat_tree: &mut BTree, - commit_id: u64, - page_ids: &[u64], -) -> Result { - if page_ids.is_empty() { - return Ok(0); - } - let dk = Catalog::deferred_free_key(); - let mut pairs = match cat_tree.get(&dk).await? { - Some(bytes) => Catalog::decode_deferred_free(&bytes)?, - None => Vec::new(), - }; - for &page_id in page_ids { - pairs.push((commit_id, page_id)); - } - // Keep sorted by commit_id for efficient draining. - pairs.sort_unstable_by_key(|(cid, _)| *cid); - let encoded = Catalog::encode_deferred_free(&pairs); - cat_tree.put(&dk, &encoded).await?; - Ok(pairs.len() as u64) -} - -/// Drain all deferred-free entries with `commit_id < min_reader_commit` into -/// the persistent free-list. Returns the number of pages promoted. -/// -/// Modifies `cat_tree` in-place. Caller flushes and commits. -pub async fn drain_deferred_to_freelist( - cat_tree: &mut BTree, - min_reader_commit: u64, -) -> Result { - let dk = Catalog::deferred_free_key(); - let pairs = match cat_tree.get(&dk).await? { - Some(bytes) => Catalog::decode_deferred_free(&bytes)?, - None => return Ok(0), - }; - - let (eligible, remaining): (Vec<_>, Vec<_>) = pairs - .into_iter() - .partition(|(cid, _)| *cid < min_reader_commit); - - let count = eligible.len() as u64; - for (_cid, page_id) in eligible { - let key = Catalog::free_list_key(page_id); - cat_tree.put(&key, &[]).await?; - } - - if remaining.is_empty() { - let _ = cat_tree.delete(&dk).await?; - } else { - let encoded = Catalog::encode_deferred_free(&remaining); - cat_tree.put(&dk, &encoded).await?; - } - - Ok(count) -} - -/// Persist the updated catalog state after free-list operations. This is a -/// low-level helper; callers that already hold the writer lock and have -/// modified a `BTree` should flush+commit themselves. Provided here for -/// completeness. -pub async fn persist_freelist_state( - _pager: &Arc>, - _realm_id: RealmId, -) -> Result<()> { - // Persistence is handled by the caller through the normal commit path. - Ok(()) -} diff --git a/src/compaction/full.rs b/src/compaction/full.rs index efc08e5..1c5533e 100644 --- a/src/compaction/full.rs +++ b/src/compaction/full.rs @@ -9,7 +9,6 @@ //! 6. Repacks segment files whose garbage ratio exceeds 5%. use crate::btree::BTree; -use crate::catalog::codec::Catalog; use crate::errors::PagedbError; use crate::pager::header::commit_header; use crate::pager::structural_header::MainDbHeaderFields; @@ -39,14 +38,20 @@ pub async fn compact_now(db: &Db) -> Result { // Acquire exclusive writer lock for the entire compact operation. let mut state = db.writer.lock().await; + // Compaction relocates and/or truncates pages, invalidating every page id + // cached for runtime reuse. Drop those reuse hints so a post-compaction + // commit can't recycle a page the repack now uses for live data. + db.free_page_cache.lock().clear(); + let old_next_page_id = state.next_page_id; - // ── 1. Check reader pins ────────────────────────────────────────────────── - // If any reader is pinned, the main tree pages are still live in their - // snapshot. We must not overwrite them with the bulk-load repack. Instead, - // only drain eligible deferred-free entries via CoW (which appends pages - // at the high end, above the reader range) and skip truncation. - let (has_readers, min_reader_commit) = { + // ── 1. Refuse while readers are pinned ─────────────────────────────────── + // A dense repack relocates the current tree and truncates the file; pinned + // readers (in-process or cross-process durable) still reference the old + // pages, so neither is safe under them. Runtime free-page reuse already + // reclaims space on ordinary commits, so there is nothing for compaction to + // do here until the readers drop. + let has_readers = { let in_mem_min = { let readers = db.tracked_readers.lock(); readers @@ -55,117 +60,12 @@ pub async fn compact_now(db: &Db) -> Result { .min() .unwrap_or(u64::MAX) }; - // Also check cross-process durable reader pins so that foreign readers - // whose pins are not in-memory are not accidentally reclaimed. let durable_min = db .min_durable_reader_commit(state.catalog_root_page_id, state.next_page_id) .await; - let min = in_mem_min.min(durable_min); - let has = min < u64::MAX; - (has, min) + in_mem_min.min(durable_min) < u64::MAX }; - if has_readers { - // Limited compaction: only drain eligible deferred-free entries into - // the persistent free-list using CoW (safe — appends above existing range). - let (cat_non_housekeeping, deferred_pairs) = - collect_catalog_split(&db.pager, db.realm_id, &state).await?; - - let (eligible_free, still_deferred): (Vec<_>, Vec<_>) = deferred_pairs - .into_iter() - .partition(|(commit_id, _)| *commit_id < min_reader_commit); - - if eligible_free.is_empty() { - // Nothing to do while readers are pinned. - return Ok(result); - } - - // Drain eligible deferred pages into the free-list using CoW. - let mut cat_tree = BTree::open( - db.pager.clone(), - db.realm_id, - state.catalog_root_page_id, - state.next_page_id, - db.page_size, - ); - - // Write free-list rows for eligible pages. - for (_cid, page_id) in &eligible_free { - let key = Catalog::free_list_key(*page_id); - cat_tree.put(&key, &[]).await?; - } - // Update the deferred-free row. - let dk = Catalog::deferred_free_key(); - if still_deferred.is_empty() { - let _ = cat_tree.delete(&dk).await; - } else { - let encoded = Catalog::encode_deferred_free(&still_deferred); - cat_tree.put(&dk, &encoded).await?; - } - - // Merge any new catalog content collected (non-housekeeping stays). - let _ = cat_non_housekeeping; - - cat_tree.flush().await?; - let new_cat_root = cat_tree.root_page_id(); - let new_next = cat_tree.next_page_id().max(state.next_page_id); - - let new_commit_id = state.latest_commit_id + 1; - let new_seq = state.seq + 1; - let counter_anchor = db.pager.pending_anchor(); - - let mut catalog_root_bytes = [0u8; 16]; - catalog_root_bytes[..8].copy_from_slice(&new_cat_root.to_le_bytes()); - catalog_root_bytes[8..].copy_from_slice(&new_commit_id.to_le_bytes()); - - let fields = MainDbHeaderFields { - format_version: 1, - cipher_id: db.cipher_id.as_byte(), - page_size_log2: page_size_log2(db.page_size)?, - flags: 0, - file_id: db.file_id, - kek_salt: db.kek_salt, - mk_epoch: db.mk_epoch.load(std::sync::atomic::Ordering::SeqCst), - seq: new_seq, - active_root_page_id: state.root_page_id, - active_root_txn_id: state.latest_commit_id, - counter_anchor, - commit_id: CommitId(new_commit_id), - free_list_root: [0u8; 16], - catalog_root: catalog_root_bytes, - apply_journal_root_page_id: 0, - apply_journal_root_version: 0, - commit_history_root_page_id: 0, - commit_history_root_version: 0, - restore_mode: 0, - next_page_id: new_next, - commit_retain_policy_tag: 0, - commit_retain_policy_value: 0, - }; - - let hk_clone = { db.hk.read().clone() }; - let new_slot = commit_header( - &*db.vfs, - &db.main_db_path, - &hk_clone, - &fields, - state.active_slot, - db.page_size, - ) - .await?; - db.pager.commit_anchor(counter_anchor)?; - - state.catalog_root_page_id = new_cat_root; - state.next_page_id = new_next; - state.active_slot = new_slot; - state.seq = new_seq; - state.latest_commit_id = new_commit_id; - state.commit_history_root_page_id = 0; - state.commit_history_root_version = 0; - db.latest_commit - .store(new_commit_id, std::sync::atomic::Ordering::SeqCst); - - result.main_db_pages_reclaimed = eligible_free.len() as u64; return Ok(result); } @@ -185,16 +85,8 @@ pub async fn compact_now(db: &Db) -> Result { Vec::new() }; - // Collect catalog pairs, separating out free-list and deferred-free rows - // so we can rebuild them correctly in the new tree. - let (cat_non_housekeeping, deferred_pairs) = - collect_catalog_split(&db.pager, db.realm_id, &state).await?; - - let (eligible_free, still_deferred): (Vec<_>, Vec<_>) = deferred_pairs - .into_iter() - .partition(|(commit_id, _)| *commit_id < min_reader_commit); - - let free_pages_before = eligible_free.len() as u64; + // Collect catalog rows (housekeeping free-list rows dropped). + let cat_rows = collect_catalog_split(&db.pager, db.realm_id, &state).await?; // ── 3. Write fresh compacted trees starting at page 4 ──────────────────── // Pages 0–3 are reserved (header slots A/B + two spares); never allocated. @@ -210,29 +102,15 @@ pub async fn compact_now(db: &Db) -> Result { let new_root = new_main.root_page_id(); let after_main = new_main.next_page_id(); - // Build new catalog tree via bulk-load. Include all non-housekeeping rows, - // plus the remaining deferred-free (if any). - // NOTE: eligible_free pages are skipped: after truncation, pages above - // new_next don't exist; pages below new_next are fully occupied by the - // compacted layout. Free-list management during normal writes is handled - // in WriteTxn::commit via free_page_deferred. - let mut cat_all: Vec<(Vec, Vec)> = cat_non_housekeeping; - if !still_deferred.is_empty() { - let dk = Catalog::deferred_free_key(); - let encoded = Catalog::encode_deferred_free(&still_deferred); - cat_all.push((dk, encoded)); - } - cat_all.sort_by(|(a, _), (b, _)| a.cmp(b)); let mut new_cat = BTree::open(db.pager.clone(), db.realm_id, 0, after_main, db.page_size); - new_cat.bulk_load(cat_all).await?; + new_cat.bulk_load(cat_rows).await?; new_cat.flush().await?; let new_cat_root = new_cat.root_page_id(); let new_next = new_cat.next_page_id(); - // Pages reclaimed = reduction in next_page_id + eligible free pages consumed. - result.main_db_pages_reclaimed = old_next_page_id - .saturating_sub(new_next) - .saturating_add(free_pages_before); + // Pages reclaimed = reduction in next_page_id (the dense layout is contiguous, + // and the durable free-list is reset to empty below). + result.main_db_pages_reclaimed = old_next_page_id.saturating_sub(new_next); // ── 4. Commit new header ───────────────────────────────────────────────── let new_commit_id = state.latest_commit_id + 1; @@ -288,6 +166,9 @@ pub async fn compact_now(db: &Db) -> Result { state.latest_commit_id = new_commit_id; state.commit_history_root_page_id = 0; state.commit_history_root_version = 0; + // The dense repack relocates/truncates every page, so the old free-list is + // gone; the new layout starts with an empty free-list. + state.free_list_root_page_id = 0; db.latest_commit .store(new_commit_id, std::sync::atomic::Ordering::SeqCst); diff --git a/src/compaction/helpers.rs b/src/compaction/helpers.rs index 1441af1..aaf5c7d 100644 --- a/src/compaction/helpers.rs +++ b/src/compaction/helpers.rs @@ -21,19 +21,17 @@ pub(super) async fn collect_all_pairs( tree.collect_range(&[], &[0xFF; 256]).await } -/// Collect catalog pairs split into two groups: -/// - `non_housekeeping`: all rows that are NOT free-list or deferred-free rows. -/// - `deferred_pairs`: the decoded deferred-free queue entries. -/// -/// Free-list rows are intentionally excluded; they will be reconstructed -/// during compaction by promoting eligible deferred-free entries. +/// Collect every catalog row, for rebuilding the catalog tree during a dense +/// repack. Free pages are tracked in the durable free-list chain (reset +/// separately by the repack), not in the catalog, so there are no housekeeping +/// rows to filter here. pub(super) async fn collect_catalog_split( pager: &Arc>, realm_id: crate::RealmId, state: &WriterState, -) -> Result<(Vec<(Vec, Vec)>, Vec<(u64, u64)>)> { +) -> Result, Vec)>> { if state.catalog_root_page_id == 0 { - return Ok((Vec::new(), Vec::new())); + return Ok(Vec::new()); } let tree = BTree::open( pager.clone(), @@ -42,28 +40,7 @@ pub(super) async fn collect_catalog_split( state.next_page_id, pager.page_size(), ); - let all = collect_all_pairs(&tree).await?; - - let fl_byte = CatalogRowKind::FreeList as u8; - let df_key = Catalog::deferred_free_key(); - - let mut non_housekeeping: Vec<(Vec, Vec)> = Vec::new(); - let mut deferred_pairs: Vec<(u64, u64)> = Vec::new(); - - for (k, v) in all { - if k.first() == Some(&fl_byte) { - // Free-list row — drop; will be rebuilt from eligible deferred entries. - continue; - } - if k == df_key { - // Deferred-free row — decode separately. - deferred_pairs = Catalog::decode_deferred_free(&v)?; - continue; - } - non_housekeeping.push((k, v)); - } - - Ok((non_housekeeping, deferred_pairs)) + collect_all_pairs(&tree).await } pub(super) async fn list_all_segments_inner( @@ -284,7 +261,15 @@ pub(super) async fn load_frontier_page_id( } } -/// Build [`MainDbHeaderFields`] for a commit. +/// Build [`MainDbHeaderFields`] for a compaction commit. +/// +/// Compaction relocates/truncates pages, so it discards the commit-history +/// index (`commit_history_root = 0`) — exactly as `compact_now` does. Writing +/// the *old* history root here would leave the durable header pointing at a +/// history tree the repack just overwrote/truncated, so a later `begin_read_at` +/// would read garbage. `free_list_root` is supplied by the caller: an +/// intermediate step preserves the still-valid chain; the final dense repack +/// passes 0 (the relocated layout starts with an empty free-list). #[allow(clippy::too_many_arguments)] pub(super) fn make_header_fields( db: &Db, @@ -295,7 +280,9 @@ pub(super) fn make_header_fields( new_root: u64, new_cat_root: u64, new_next: u64, + free_list_root_page_id: u64, ) -> MainDbHeaderFields { + let _ = state; let mut catalog_root_bytes = [0u8; 16]; catalog_root_bytes[..8].copy_from_slice(&new_cat_root.to_le_bytes()); catalog_root_bytes[8..].copy_from_slice(&new_commit_id.to_le_bytes()); @@ -312,12 +299,12 @@ pub(super) fn make_header_fields( active_root_txn_id: new_commit_id, counter_anchor, commit_id: CommitId(new_commit_id), - free_list_root: [0u8; 16], + free_list_root: crate::txn::db::encode_free_list_root(free_list_root_page_id), catalog_root: catalog_root_bytes, apply_journal_root_page_id: 0, apply_journal_root_version: 0, - commit_history_root_page_id: state.commit_history_root_page_id, - commit_history_root_version: state.commit_history_root_version, + commit_history_root_page_id: 0, + commit_history_root_version: 0, restore_mode: 0, next_page_id: new_next, commit_retain_policy_tag: 0, diff --git a/src/compaction/mod.rs b/src/compaction/mod.rs index ecc584a..3bfc431 100644 --- a/src/compaction/mod.rs +++ b/src/compaction/mod.rs @@ -1,18 +1,17 @@ -//! Online compaction: persistent free-list management, main.db defragmentation, -//! and segment repacking. +//! Online compaction: main.db defragmentation and segment repacking. +//! +//! Free-page reclamation itself is handled at runtime by the durable free-list +//! ([`crate::pager::freelist`]); compaction's job is the dense repack + file +//! truncation and segment garbage collection. //! //! Entry points: [`compact_now`] (full one-shot) and [`compact_step`] //! (incremental, budget-bounded) on a [`crate::Db`] handle. -pub mod freelist; mod full; mod helpers; mod step; mod types; -pub use freelist::{ - alloc_page, drain_deferred_to_freelist, free_page_deferred, persist_freelist_state, -}; pub use full::compact_now; pub use step::compact_step; pub use types::{CompactBudget, CompactProgress, CompactStats}; diff --git a/src/compaction/step.rs b/src/compaction/step.rs index 8dbb78a..780fef6 100644 --- a/src/compaction/step.rs +++ b/src/compaction/step.rs @@ -8,7 +8,6 @@ use crate::pager::header::commit_header; use crate::txn::db::Db; use crate::vfs::{Vfs, VfsFile}; -use super::freelist::drain_deferred_to_freelist; use super::helpers::{ collect_all_pairs, collect_catalog_split, collect_range_limited, load_compaction_state, load_frontier_page_id, make_header_fields, next_key_after, @@ -35,8 +34,14 @@ pub async fn compact_step( let mut state = db.writer.lock().await; - // ── Determine reader presence ───────────────────────────────────────────── - let (has_readers, min_reader_commit) = { + // Relocation/truncation below invalidates cached reuse page ids; drop them. + db.free_page_cache.lock().clear(); + + // ── Refuse while readers are pinned ─────────────────────────────────────── + // Relocation/truncation is unsafe under a pinned reader (in-process or + // cross-process durable). Runtime free-page reuse reclaims space on ordinary + // commits, so compaction simply waits for the readers to drop. + let has_readers = { let in_mem_min = { let readers = db.tracked_readers.lock(); readers @@ -48,89 +53,12 @@ pub async fn compact_step( let durable_min = db .min_durable_reader_commit(state.catalog_root_page_id, state.next_page_id) .await; - let min = in_mem_min.min(durable_min); - let has = min < u64::MAX; - (has, min) + in_mem_min.min(durable_min) < u64::MAX }; - if has_readers { - // With readers pinned we can only drain eligible deferred-free entries. - let (cat_non_housekeeping, deferred_pairs) = - collect_catalog_split(&db.pager, db.realm_id, &state).await?; - let (eligible_free, still_deferred): (Vec<_>, Vec<_>) = deferred_pairs - .into_iter() - .partition(|(commit_id, _)| *commit_id < min_reader_commit); - - if eligible_free.is_empty() { - let wm = load_frontier_page_id(&db.pager, db.realm_id, &state).await; - return Ok(CompactProgress { - pages_relocated: 0, - bytes_freed: 0, - more_work: true, - watermark: wm, - }); - } - - let mut cat_tree = BTree::open( - db.pager.clone(), - db.realm_id, - state.catalog_root_page_id, - state.next_page_id, - db.page_size, - ); - for (_cid, page_id) in &eligible_free { - let key = Catalog::free_list_key(*page_id); - cat_tree.put(&key, &[]).await?; - } - let dk = Catalog::deferred_free_key(); - if still_deferred.is_empty() { - let _ = cat_tree.delete(&dk).await; - } else { - let encoded = Catalog::encode_deferred_free(&still_deferred); - cat_tree.put(&dk, &encoded).await?; - } - let _ = cat_non_housekeeping; - cat_tree.flush().await?; - let new_cat_root = cat_tree.root_page_id(); - let new_next = cat_tree.next_page_id().max(state.next_page_id); - - let new_commit_id = state.latest_commit_id + 1; - let new_seq = state.seq + 1; - let counter_anchor = db.pager.pending_anchor(); - let fields = make_header_fields( - db, - &state, - new_commit_id, - new_seq, - counter_anchor, - state.root_page_id, - new_cat_root, - new_next, - ); - let hk_clone = { db.hk.read().clone() }; - let new_slot = commit_header( - &*db.vfs, - &db.main_db_path, - &hk_clone, - &fields, - state.active_slot, - db.page_size, - ) - .await?; - db.pager.commit_anchor(counter_anchor)?; - state.catalog_root_page_id = new_cat_root; - state.next_page_id = new_next; - state.active_slot = new_slot; - state.seq = new_seq; - state.latest_commit_id = new_commit_id; - state.commit_history_root_page_id = 0; - state.commit_history_root_version = 0; - db.latest_commit - .store(new_commit_id, std::sync::atomic::Ordering::SeqCst); - let wm = load_frontier_page_id(&db.pager, db.realm_id, &state).await; return Ok(CompactProgress { - pages_relocated: eligible_free.len() as u64, + pages_relocated: 0, bytes_freed: 0, more_work: true, watermark: wm, @@ -202,14 +130,7 @@ pub async fn compact_step( Vec::new() }; - let (cat_non_housekeeping, deferred_pairs) = - collect_catalog_split(&db.pager, db.realm_id, &state).await?; - - let (eligible_free, still_deferred): (Vec<_>, Vec<_>) = deferred_pairs - .into_iter() - .partition(|(commit_id, _)| *commit_id < min_reader_commit); - - let free_pages_before = eligible_free.len() as u64; + let cat_rows = collect_catalog_split(&db.pager, db.realm_id, &state).await?; let mut new_main = BTree::open(db.pager.clone(), db.realm_id, 0, 4, db.page_size); new_main.bulk_load(main_pairs).await?; @@ -219,15 +140,10 @@ pub async fn compact_step( // Remove compaction-state row from the catalog being rebuilt. let cs_key_prefix = crate::catalog::codec::CatalogRowKind::CompactionState as u8; - let mut cat_all: Vec<(Vec, Vec)> = cat_non_housekeeping + let mut cat_all: Vec<(Vec, Vec)> = cat_rows .into_iter() .filter(|(k, _)| k.first() != Some(&cs_key_prefix)) .collect(); - if !still_deferred.is_empty() { - let dk = Catalog::deferred_free_key(); - let encoded = Catalog::encode_deferred_free(&still_deferred); - cat_all.push((dk, encoded)); - } cat_all.sort_by(|(a, _), (b, _)| a.cmp(b)); let mut new_cat = BTree::open(db.pager.clone(), db.realm_id, 0, after_main, db.page_size); new_cat.bulk_load(cat_all).await?; @@ -235,13 +151,12 @@ pub async fn compact_step( let new_cat_root = new_cat.root_page_id(); let new_next = new_cat.next_page_id(); - let pages_reclaimed = old_next_page_id - .saturating_sub(new_next) - .saturating_add(free_pages_before); + let pages_reclaimed = old_next_page_id.saturating_sub(new_next); let new_commit_id = state.latest_commit_id + 1; let new_seq = state.seq + 1; let counter_anchor = db.pager.pending_anchor(); + // Dense repack: the relocated layout starts with an empty free-list. let fields = make_header_fields( db, &state, @@ -251,6 +166,7 @@ pub async fn compact_step( new_root, new_cat_root, new_next, + 0, ); let hk_clone = { db.hk.read().clone() }; let new_slot = commit_header( @@ -272,6 +188,7 @@ pub async fn compact_step( state.latest_commit_id = new_commit_id; state.commit_history_root_page_id = 0; state.commit_history_root_version = 0; + state.free_list_root_page_id = 0; db.latest_commit .store(new_commit_id, std::sync::atomic::Ordering::SeqCst); @@ -301,7 +218,6 @@ pub async fn compact_step( .last() .map_or_else(|| frontier_key.clone(), |(k, _)| k.clone()); - // Build catalog tree with deferred-free drained, then update watermark. let mut cat_tree = BTree::open( db.pager.clone(), db.realm_id, @@ -309,12 +225,11 @@ pub async fn compact_step( state.next_page_id, db.page_size, ); - // Drain all deferred-free into free-list (no readers pinned). - drain_deferred_to_freelist(&mut cat_tree, u64::MAX).await?; // Re-insert the batch: delete+put forces page reallocation to low-address - // free slots. Pages freed by delete are added to the free-list immediately - // (no readers), so the subsequent put can reuse them. + // free slots. Pages freed by the delete are recycled within this same txn + // (the allocator pops the per-session freed list), so the subsequent put + // reuses them — densifying the layout into the low page-id range. let mut main_tree = BTree::open( db.pager.clone(), db.realm_id, @@ -327,13 +242,6 @@ pub async fn compact_step( main_tree.put(k, v).await?; } - // Freed pages from main_tree deletions: add to catalog free-list directly. - let freed_by_tree = main_tree.drain_freed(); - for pid in freed_by_tree { - let key = Catalog::free_list_key(pid); - cat_tree.put(&key, &[]).await?; - } - main_tree.flush().await?; // main_tree.flush() may allocate pages while materializing the dirty-leaf // cache. Sync the catalog tree forward so its flush doesn't reuse a @@ -371,6 +279,10 @@ pub async fn compact_step( let new_commit_id = state.latest_commit_id + 1; let new_seq = state.seq + 1; let counter_anchor = db.pager.pending_anchor(); + // An intermediate step only relocates main-tree pages; it touches neither + // the durable free-list chain nor its free pages, so the free-list stays + // valid and is preserved across the step (the pre-existing free pages remain + // reusable by ordinary writes). The final dense-repack batch resets it. let fields = make_header_fields( db, &state, @@ -380,6 +292,7 @@ pub async fn compact_step( new_main_root, final_cat_root, final_next, + state.free_list_root_page_id, ); let hk_clone = { db.hk.read().clone() }; let new_slot = commit_header( diff --git a/src/options.rs b/src/options.rs index 724e3c0..46bece6 100644 --- a/src/options.rs +++ b/src/options.rs @@ -105,32 +105,6 @@ pub struct OpenOptions { /// this many newly-encrypted pages — exceed it and the txn aborts. Large /// bulk loads need a larger budget. Default: 1024. pub anchor_budget: u64, - - /// **Pagedb extension beyond the architecture spec.** When `true` and no - /// readers are pinned at commit time, skip persisting freed pages into - /// the deferred-free queue / persistent free-list. The pages become - /// **orphan pages** in `main.db` — physically allocated but unreferenced - /// — and can only be reclaimed by [`Db::compact`]. - /// - /// Trade-off: - /// - **Pro**: Eliminates one catalog-tree `CoW` per commit. Big - /// single-put write-latency win (~75µs on a 4 KiB page DB). - /// - **Con**: `main.db` grows monotonically during no-reader phases. - /// Embedders MUST schedule periodic [`Db::compact`] to reclaim space, - /// or accept unbounded growth. - /// - /// Default: `false` (spec-conformant; every freed page is tracked). - /// - /// Use cases that should enable this: - /// - Benchmarks measuring against engines without an equivalent free-list - /// - Ephemeral / cache-like deployments that periodically compact or - /// rebuild - /// - Workloads where the embedder can prove the file is short-lived - /// - /// Use cases that should NOT enable this: - /// - Long-running write-heavy stores without compaction infrastructure - /// - Anything where unbounded `main.db` growth is a problem - pub skip_freelist_persistence_when_no_readers: bool, } impl Default for OpenOptions { @@ -145,9 +119,6 @@ impl Default for OpenOptions { observer_retry_count: 3, metrics_enabled: true, anchor_budget: crate::crypto::nonce::DEFAULT_ANCHOR_BUDGET, - // Default: spec-conformant. Embedders that want the fast path - // explicitly opt in. - skip_freelist_persistence_when_no_readers: false, } } } @@ -218,14 +189,4 @@ impl OpenOptions { self.anchor_budget = v; self } - - /// Opt into the no-readers fast-free path. See - /// [`Self::skip_freelist_persistence_when_no_readers`] for the trade-off. - /// Embedders that enable this **must** schedule periodic - /// [`Db::compact`](crate::Db::compact) to reclaim orphan pages. - #[must_use] - pub fn with_skip_freelist_persistence_when_no_readers(mut self, v: bool) -> Self { - self.skip_freelist_persistence_when_no_readers = v; - self - } } diff --git a/src/pager/freelist.rs b/src/pager/freelist.rs new file mode 100644 index 0000000..a6e6e1c --- /dev/null +++ b/src/pager/freelist.rs @@ -0,0 +1,164 @@ +//! Durable free-page list, stored as a chain of `PageKind::Free` pages rooted +//! at the A/B header's `free_list_root` slot — outside the catalog B+ tree. +//! +//! Keeping the free list out of the catalog is what makes free-page recycling +//! both **durable** (it survives an unclean shutdown — the chain is committed +//! atomically with the header swap) and **bounded** (maintaining it never +//! copies-on-writes the catalog tree, so it adds no per-commit catalog churn). +//! +//! Each entry is a `(commit_id, page_id)` pair: `commit_id` is the commit that +//! freed the page, used at `begin_write` to decide which pages are below the +//! reclamation floor (observable by no reader and no retained-history root) and +//! therefore safe to recycle now. The chain stores *every* free page — those +//! still pinned are simply carried forward until the floor advances past them. +//! +//! Page body layout (`PageKind::Free`): +//! ```text +//! [0..8) next chain page id (LE u64; 0 = end of chain) +//! [8..12) entry count in this page (LE u32) +//! [12..) `count` × (commit_id LE u64 ‖ page_id LE u64) +//! ``` + +use std::collections::HashSet; + +use crate::pager::Pager; +use crate::pager::format::data_page::body_capacity; +use crate::pager::format::page_kind::PageKind; +use crate::vfs::Vfs; +use crate::{PagedbError, RealmId, Result}; + +const ENTRY_LEN: usize = 16; +const PAGE_HEADER_LEN: usize = 12; + +/// Number of `(commit_id, page_id)` entries one free-list page can hold. +#[must_use] +pub const fn chain_capacity(page_size: usize) -> usize { + (body_capacity(page_size) - PAGE_HEADER_LEN) / ENTRY_LEN +} + +/// Walk the free-list chain from `head`, returning all `(commit_id, page_id)` +/// entries and the list of page ids the chain itself occupies. `head == 0` is +/// an empty chain. +pub async fn read_chain( + pager: &Pager, + realm_id: RealmId, + head: u64, +) -> Result<(Vec<(u64, u64)>, Vec)> { + let mut entries = Vec::new(); + let mut chain_pages = Vec::new(); + let mut page = head; + while page != 0 { + let guard = pager.read_main_page(page, realm_id, PageKind::Free).await?; + let body = guard.body_ref(); + let mut next_b = [0u8; 8]; + next_b.copy_from_slice(&body[0..8]); + let next = u64::from_le_bytes(next_b); + let mut cnt_b = [0u8; 4]; + cnt_b.copy_from_slice(&body[8..12]); + let count = u32::from_le_bytes(cnt_b) as usize; + for i in 0..count { + let off = PAGE_HEADER_LEN + i * ENTRY_LEN; + let mut cid_b = [0u8; 8]; + cid_b.copy_from_slice(&body[off..off + 8]); + let mut pid_b = [0u8; 8]; + pid_b.copy_from_slice(&body[off + 8..off + 16]); + entries.push((u64::from_le_bytes(cid_b), u64::from_le_bytes(pid_b))); + } + chain_pages.push(page); + page = next; + } + Ok((entries, chain_pages)) +} + +/// Persist `entries` as a fresh chain, returning the new head page id and the +/// updated `next_page` cursor. +/// +/// Chain pages are drawn first from `host_candidates` — pages that are already +/// free and observable by no snapshot, hence safe to overwrite — and only then +/// bump-allocated from `next_page`. A carved host is removed from the persisted +/// entries (it now holds the chain itself). The caller MUST ensure +/// `host_candidates` are a subset of `entries`' pages and disjoint from any +/// live page or the old chain's own pages (which must stay readable until the +/// header swap). +pub async fn rewrite_chain( + pager: &Pager, + realm_id: RealmId, + page_size: usize, + mut entries: Vec<(u64, u64)>, + host_candidates: Vec, + next_page: u64, +) -> Result<(u64, u64)> { + let cap = chain_capacity(page_size); + let total = entries.len(); + let mut next = next_page; + let mut carved: HashSet = HashSet::new(); + let mut chain_pages: Vec = Vec::new(); + let mut hosts = host_candidates.into_iter(); + loop { + let remaining = total - carved.len(); + let need = if remaining == 0 { + 0 + } else { + remaining.div_ceil(cap) + }; + if chain_pages.len() >= need { + break; + } + if let Some(h) = hosts.next() { + carved.insert(h); + chain_pages.push(h); + } else { + chain_pages.push(next); + next += 1; + } + } + entries.retain(|(_, pid)| !carved.contains(pid)); + let head = write_chain(pager, realm_id, page_size, &chain_pages, &entries).await?; + Ok((head, next)) +} + +/// Write `entries` across the supplied `chain_pages` (which must provide enough +/// capacity: `chain_pages.len() * chain_capacity(page_size) >= entries.len()`), +/// linking them into a chain. Returns the new head page id, or `0` when there +/// is nothing to write. The pages are inserted into the pager's dirty set; the +/// caller flushes and commits the header (carrying the returned head). +pub async fn write_chain( + pager: &Pager, + realm_id: RealmId, + page_size: usize, + chain_pages: &[u64], + entries: &[(u64, u64)], +) -> Result { + if entries.is_empty() { + return Ok(0); + } + let cap = chain_capacity(page_size); + let body_len = body_capacity(page_size); + debug_assert!(chain_pages.len() * cap >= entries.len()); + let mut written = 0usize; + for (i, &page_id) in chain_pages.iter().enumerate() { + let chunk = &entries[written..(written + cap).min(entries.len())]; + let next = chain_pages.get(i + 1).copied().unwrap_or(0); + // The last page used carries 0 as `next` even if more pages were + // supplied than needed (they go unused). + let next = if chunk.is_empty() { 0 } else { next }; + let mut body = vec![0u8; body_len]; + body[0..8].copy_from_slice(&next.to_le_bytes()); + let chunk_len = u32::try_from(chunk.len()) + .map_err(|_| PagedbError::Io(std::io::Error::other("free-list chunk_len overflow")))?; + body[8..12].copy_from_slice(&chunk_len.to_le_bytes()); + for (j, (cid, pid)) in chunk.iter().enumerate() { + let off = PAGE_HEADER_LEN + j * ENTRY_LEN; + body[off..off + 8].copy_from_slice(&cid.to_le_bytes()); + body[off + 8..off + 16].copy_from_slice(&pid.to_le_bytes()); + } + pager + .write_main_page(page_id, realm_id, PageKind::Free, &body) + .await?; + written += chunk.len(); + if written >= entries.len() { + break; + } + } + Ok(chain_pages[0]) +} diff --git a/src/pager/mod.rs b/src/pager/mod.rs index 4661180..ada2747 100644 --- a/src/pager/mod.rs +++ b/src/pager/mod.rs @@ -3,6 +3,7 @@ pub mod cache; pub mod core; pub mod format; +pub mod freelist; pub mod header; pub use cache::PageCache; diff --git a/src/recovery/deep_walk.rs b/src/recovery/deep_walk.rs index a9707dc..a0f30c8 100644 --- a/src/recovery/deep_walk.rs +++ b/src/recovery/deep_walk.rs @@ -146,12 +146,13 @@ impl DeepWalkReport { pub async fn run_deep_walk(db: &Db) -> Result { let mut report = DeepWalkReport::default(); - let (next_page_id, catalog_root, catalog_next) = { + let (next_page_id, catalog_root, catalog_next, free_list_root) = { let state = db.writer.lock().await; ( state.next_page_id, state.catalog_root_page_id, state.next_page_id, + state.free_list_root_page_id, ) }; @@ -163,7 +164,51 @@ pub async fn run_deep_walk(db: &Db) -> Result let realm_id = db.realm_id; // Collect the set of page IDs reachable from all live roots. - let reachable = collect_reachable_pages(db).await; + let mut reachable = collect_reachable_pages(db).await; + + // Walk and validate the durable free-list chain. Reading it verifies each + // chain page's AEAD; a corrupt chain surfaces as a page issue. Validate the + // entries (in range, unique, and not also live), then account the chain's + // own pages and the free pages it tracks so they are not reported as + // orphans — they are legitimately owned by the free-list, not stray. + match crate::pager::freelist::read_chain(&db.pager, realm_id, free_list_root).await { + Ok((free_entries, chain_pages)) => { + let mut seen: BTreeSet = BTreeSet::new(); + for &(_cid, pid) in &free_entries { + if pid >= next_page_id { + report.page_issues.push(PageIssue { + page_id: pid, + description: "free-list entry references a page past next_page_id" + .to_string(), + }); + } else if reachable.contains(&pid) { + report.page_issues.push(PageIssue { + page_id: pid, + description: "page is both live (reachable from a root) and free-listed" + .to_string(), + }); + } + if !seen.insert(pid) { + report.page_issues.push(PageIssue { + page_id: pid, + description: "duplicate free-list entry".to_string(), + }); + } + } + for p in chain_pages { + reachable.insert(p); + } + for (_cid, pid) in free_entries { + reachable.insert(pid); + } + } + Err(e) => { + report.page_issues.push(PageIssue { + page_id: free_list_root, + description: format!("free-list chain unreadable: {e}"), + }); + } + } let vfs: &V = &db.vfs; let main_file_res = vfs.open(main_db_path, OpenMode::Read).await; diff --git a/src/txn/db/catalog.rs b/src/txn/db/catalog.rs index 536fbd0..efa4cd2 100644 --- a/src/txn/db/catalog.rs +++ b/src/txn/db/catalog.rs @@ -15,6 +15,42 @@ use super::core::{ }; impl Db { + /// The oldest commit id still retained in the commit-history index, or + /// `None` when history is disabled or the index is empty. Pages reachable + /// from this commit's root (or any newer one) must not be recycled, so it + /// is one of the two floors gating free-page reclamation (the other is the + /// oldest live reader pin). Reads only the leftmost spine of the history + /// tree — O(height), not O(retained count). + pub(crate) async fn oldest_retained_history_commit( + &self, + commit_history_root_page_id: u64, + next_page_id: u64, + ) -> Result> { + if matches!( + self.options.commit_history_retain, + crate::options::RetainPolicy::Disabled + ) || commit_history_root_page_id == 0 + { + return Ok(None); + } + let hist = BTree::open( + self.pager.clone(), + self.realm_id, + commit_history_root_page_id, + next_page_id, + self.page_size, + ); + let Some(key) = hist.first_key().await? else { + return Ok(None); + }; + if key.len() < 8 { + return Ok(None); + } + let mut b = [0u8; 8]; + b.copy_from_slice(&key[..8]); + Ok(Some(u64::from_be_bytes(b))) + } + /// Scan catalog counter rows and bump any whose stored value is less than /// `anchor` up to `anchor`. Called once at open to recover monotonicity /// after a torn write. Errors are silently ignored (best-effort); a failure @@ -87,6 +123,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: state.commit_history_root_page_id, commit_history_root_version: state.commit_history_root_version, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; let hk_clone = self.hk.read().clone(); @@ -138,6 +175,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; let hk_clone = { self.hk.read().clone() }; @@ -180,16 +218,17 @@ impl Db { } } - /// Insert a commit-history entry into the commit-history B+ tree, prune - /// according to the retention policy, and return the updated - /// `(root_page_id, root_version, new_next_page_id)`. + /// Insert the new commit-history entry and prune per the retention policy. + /// Returns the page ids freed by this tree's copy-on-write and pruning, so + /// the caller can hand them to the shared allocator cache for reuse (they + /// are never reader-pinned). #[allow(clippy::too_many_lines)] pub(crate) async fn write_commit_history_entry( &self, state: &mut WriterState, new_commit_id: u64, meta: CommitHistoryMeta, - ) -> Result<()> { + ) -> Result> { let min_pinned = { let readers = self.tracked_readers.lock(); readers.iter().map(|r| r.commit_id.0).min() @@ -202,6 +241,14 @@ impl Db { state.next_page_id, self.page_size, ); + // The commit-history tree is not part of any reader's pinned snapshot + // (readers track the data and catalog roots, never the history root), so + // every page its copy-on-write/prune frees is immediately reusable. + // Recycle freely and feed the shared allocator cache so per-commit + // history churn does not leak pages over a long-lived writer's lifetime. + hist_tree.set_reuse_threshold(0); + hist_tree.set_free_page_cache(self.free_page_cache.clone()); + hist_tree.set_free_page_consumed(self.free_page_consumed.clone()); // Insert the new entry. let key = new_commit_id.to_be_bytes().to_vec(); @@ -322,6 +369,13 @@ impl Db { // commit's unified `pager.flush_main` picks them up) without issuing a // separate fsync. The caller is responsible for flushing the pager. hist_tree.materialize_dirty().await?; + // Capture spine/prune frees after materialization (they are realized + // during the flush, not before it). + let freed: Vec = hist_tree + .drain_freed() + .into_iter() + .filter(|&p| p >= 4) + .collect(); let new_hist_root = hist_tree.root_page_id(); let new_next = hist_tree.next_page_id().max(state.next_page_id); @@ -329,6 +383,6 @@ impl Db { state.commit_history_root_version = new_commit_id; state.next_page_id = new_next; - Ok(()) + Ok(freed) } } diff --git a/src/txn/db/core.rs b/src/txn/db/core.rs index 7930dff..22d830e 100644 --- a/src/txn/db/core.rs +++ b/src/txn/db/core.rs @@ -48,6 +48,10 @@ pub(crate) struct WriterState { pub seq: u64, pub catalog_root_page_id: u64, pub catalog_root_txn_id: u64, + /// Head page id of the durable free-list chain (0 = empty). Stored in the + /// A/B header's `free_list_root` slot and rewritten atomically with each + /// commit. See [`crate::pager::freelist`]. + pub free_list_root_page_id: u64, /// Root page id of the commit-history B+ tree (0 = not yet created). pub commit_history_root_page_id: u64, /// Version / `txn_id` of the commit-history tree's last write. @@ -117,14 +121,18 @@ pub struct Db { /// `PinHandle` so durable-pin commit paths can also publish. pub(crate) snapshot: Arc>, /// Cross-commit cache of page IDs known to be safely reusable. Populated - /// on commit when `skip_freelist_persistence_when_no_readers` is enabled - /// and no readers were pinned: instead of orphaning freed pages, recycle - /// them in-memory so the next writer txn's `allocate_page` pops from - /// here before bumping `next_page_id`. Keeps the file size bounded under - /// the fast-free option. Shared with each session's `BTree` via the same - /// `Arc` so all three trees in a txn (main, catalog, history) draw from - /// the same pool. + /// after each commit with the pages it freed that no live reader and no + /// retained commit-history root can still observe; the next writer txn's + /// `allocate_page` pops from here before bumping `next_page_id`, keeping the + /// file size bounded under sustained writes. Shared with each session's + /// `BTree` via the same `Arc` so all three trees in a txn (main, catalog, + /// history) draw from the same pool. Cleared by `compact_now`'s full repack, + /// which relocates pages and invalidates every cached id. pub(crate) free_page_cache: Arc>>, + /// Per-txn sink (cleared at `begin_write`) recording page ids the allocator + /// drew from `free_page_cache`. The commit path removes them from the + /// durable free-list — they now hold live committed data. + pub(crate) free_page_consumed: Arc>>, } /// Reader-visible state, refreshed by the writer at commit time. @@ -146,6 +154,23 @@ pub(super) fn encode_root_ref(page_id: u64, txn_id: u64) -> [u8; 16] { bytes } +/// Encode the header's `free_list_root` slot from the free-list chain head page +/// id (low 8 bytes, LE; remaining bytes reserved/zero). +#[must_use] +pub(crate) fn encode_free_list_root(head_page_id: u64) -> [u8; 16] { + let mut bytes = [0u8; 16]; + bytes[..8].copy_from_slice(&head_page_id.to_le_bytes()); + bytes +} + +/// Decode the free-list chain head page id from the header's `free_list_root`. +#[must_use] +pub(crate) fn decode_free_list_root(raw: &[u8; 16]) -> u64 { + let mut b = [0u8; 8]; + b.copy_from_slice(&raw[..8]); + u64::from_le_bytes(b) +} + /// Variable fields supplied per call site when assembling a /// [`MainDbHeaderFields`] for a writer header commit. The constant fields /// (`format_version`, `flags`, identity bytes, and the always-zero @@ -163,6 +188,10 @@ pub(super) struct HeaderFieldsParams { pub commit_history_root_page_id: u64, pub commit_history_root_version: u64, pub next_page_id: u64, + /// Head page id of the durable free-list chain to record in the header. + /// Header writes that don't touch the free list pass the current + /// `state.free_list_root_page_id` so it is preserved across the swap. + pub free_list_root_page_id: u64, } impl Db { @@ -185,7 +214,7 @@ impl Db { active_root_txn_id: params.active_root_txn_id, counter_anchor: params.counter_anchor, commit_id: CommitId(params.commit_id), - free_list_root: [0; 16], + free_list_root: encode_free_list_root(params.free_list_root_page_id), catalog_root: params.catalog_root, apply_journal_root_page_id: 0, apply_journal_root_version: 0, diff --git a/src/txn/db/misc.rs b/src/txn/db/misc.rs index 11ff4b0..d9e5ae1 100644 --- a/src/txn/db/misc.rs +++ b/src/txn/db/misc.rs @@ -96,16 +96,23 @@ impl Db { use std::sync::atomic::Ordering as AtOrd; // Gather writer-guarded values. - let (latest_commit_id, next_page_id, catalog_root, catalog_next) = { + let (latest_commit_id, next_page_id, catalog_root, catalog_next, free_list_root) = { let w = self.writer.lock().await; ( w.latest_commit_id, w.next_page_id, w.catalog_root_page_id, w.next_page_id, + w.free_list_root_page_id, ) }; + // Durable free-list depth (chain rooted at the header's free_list_root). + let free_list_pending_entries = + crate::pager::freelist::read_chain(&self.pager, self.realm_id, free_list_root) + .await + .map_or(0, |(entries, _)| entries.len() as u64); + // Main database file size. let main_db_bytes = match self .vfs @@ -137,10 +144,9 @@ impl Db { let pending_tombstones = u32::try_from(self.pending_tombstones.lock().len()).unwrap_or(u32::MAX); - // Segment stats and free-list counts from catalog. - let (segments_live, segments_total_bytes, free_list_pending_entries) = if catalog_root == 0 - { - (0u32, 0u64, 0u64) + // Segment stats from catalog. + let (segments_live, segments_total_bytes) = if catalog_root == 0 { + (0u32, 0u64) } else { let tree = BTree::open( self.pager.clone(), @@ -150,7 +156,6 @@ impl Db { self.page_size, ); - // Segments. let seg_start = vec![CatalogRowKind::Segment as u8]; let mut seg_end = seg_start.clone(); seg_end.push(0xFF); @@ -165,26 +170,7 @@ impl Db { .map(|m| m.total_bytes) .sum(); - // Free-list entries (persistent free-list rows). - let fl_start = vec![CatalogRowKind::FreeList as u8]; - let mut fl_end = fl_start.clone(); - fl_end.push(0xFF); - let fl_rows = tree - .collect_range(&fl_start, &fl_end) - .await - .unwrap_or_default(); - let fl_count = fl_rows.len() as u64; - - // Deferred-free entries. - let df_key = Catalog::deferred_free_key(); - let df_count = match tree.get(&df_key).await { - Ok(Some(bytes)) => { - Catalog::decode_deferred_free(&bytes).map_or(0, |v| v.len() as u64) - } - _ => 0, - }; - - (seg_count, seg_bytes, fl_count + df_count) + (seg_count, seg_bytes) }; Ok(DbStats { diff --git a/src/txn/db/mod.rs b/src/txn/db/mod.rs index e1bc9c7..d42c85a 100644 --- a/src/txn/db/mod.rs +++ b/src/txn/db/mod.rs @@ -14,4 +14,6 @@ mod snapshot; mod util; pub use core::Db; -pub(crate) use core::{CommitHistoryMeta, PendingTombstone, ReaderSnapshot, WriterState}; +pub(crate) use core::{ + CommitHistoryMeta, PendingTombstone, ReaderSnapshot, WriterState, encode_free_list_root, +}; diff --git a/src/txn/db/open.rs b/src/txn/db/open.rs index 81efccd..7437502 100644 --- a/src/txn/db/open.rs +++ b/src/txn/db/open.rs @@ -152,6 +152,7 @@ impl Db { seq: 1, catalog_root_page_id: 0, catalog_root_txn_id: 0, + free_list_root_page_id: 0, commit_history_root_page_id: 0, commit_history_root_version: 0, // Fresh DB: the commit-history tree is empty. @@ -191,6 +192,7 @@ impl Db { catalog_root_page_id: 0, })), free_page_cache: Arc::new(parking_lot::Mutex::new(Vec::new())), + free_page_consumed: Arc::new(parking_lot::Mutex::new(Vec::new())), }) } @@ -202,7 +204,7 @@ impl Db { realm: RealmId, options: OpenOptions, ) -> Result { - Self::open_existing_inner(vfs, kek, page_size, realm, options).await + Self::open_existing_inner(vfs, kek, page_size, realm, options, DbMode::Standalone).await } /// Open an existing database that was previously created with @@ -214,7 +216,15 @@ impl Db { page_size: usize, realm: RealmId, ) -> Result { - Self::open_existing_inner(vfs, kek, page_size, realm, OpenOptions::default()).await + Self::open_existing_inner( + vfs, + kek, + page_size, + realm, + OpenOptions::default(), + DbMode::Standalone, + ) + .await } #[allow(clippy::too_many_lines)] @@ -224,6 +234,7 @@ impl Db { page_size: usize, realm: RealmId, options: OpenOptions, + mode: DbMode, ) -> Result { use crate::vfs::VfsFile; use crate::vfs::types::OpenMode; @@ -327,6 +338,7 @@ impl Db { seq: fields.seq, catalog_root_page_id, catalog_root_txn_id, + free_list_root_page_id: super::core::decode_free_list_root(&fields.free_list_root), commit_history_root_page_id: fields.commit_history_root_page_id, commit_history_root_version: fields.commit_history_root_version, // Lazily populated on first `write_commit_history_entry` call. @@ -384,7 +396,7 @@ impl Db { mmap_bytes_in_use: std::sync::Arc::new(AtomicU64::new(0)), spill_bytes_in_use: AtomicU64::new(0), txn_seq: AtomicU64::new(0), - mode: DbMode::Standalone, + mode, aborted_readers: parking_lot::Mutex::new(std::collections::HashSet::new()), sentinel_locks: Vec::new(), snapshot: Arc::new(parking_lot::RwLock::new(ReaderSnapshot { @@ -394,6 +406,7 @@ impl Db { catalog_root_page_id, })), free_page_cache: Arc::new(parking_lot::Mutex::new(Vec::new())), + free_page_consumed: Arc::new(parking_lot::Mutex::new(Vec::new())), }; // If an online rekey was interrupted (crash or abort), resume and @@ -407,29 +420,34 @@ impl Db { } } - // Remove stale and expired reader-pin rows: own-PID rows from a prior - // process incarnation (crash without cleanup) and rows whose - // expires_unix_seconds is in the past. - { - let mut state = db.writer.lock().await; - let hk_clone = db.hk.read().clone(); - let _ = cleanup_stale_reader_pins( - &db.pager, - &db.vfs, - &db.main_db_path, - &hk_clone, - db.realm_id, - db.page_size, - db.cipher_id, - db.file_id, - db.kek_salt, - db.mk_epoch.load(Ordering::SeqCst), - &mut state, - ) - .await; - // Snapshot may have advanced by the bulk-pin-cleanup commit - // performed inside cleanup_stale_reader_pins. - db.publish_snapshot(&state); + // Recovery work that mutates the catalog/header is only valid for + // handle modes that hold the exclusive writer lock. ReadOnly / Observer + // must never write, so they skip pin cleanup and backlog draining. + if matches!(mode, DbMode::Standalone | DbMode::Follower) { + // Remove every durable reader-pin row left behind by a prior + // (now-dead) incarnation. Safe to clear all of them while holding + // the exclusive writer lock — see `cleanup_stale_reader_pins`. + { + let mut state = db.writer.lock().await; + let hk_clone = db.hk.read().clone(); + let _ = cleanup_stale_reader_pins( + &db.pager, + &db.vfs, + &db.main_db_path, + &hk_clone, + db.realm_id, + db.page_size, + db.cipher_id, + db.file_id, + db.kek_salt, + db.mk_epoch.load(Ordering::SeqCst), + &mut state, + ) + .await; + // Snapshot may have advanced by the bulk-pin-cleanup commit + // performed inside cleanup_stale_reader_pins. + db.publish_snapshot(&state); + } } // Durable monotonicity recovery for named counters. @@ -564,9 +582,11 @@ impl Db { } } - // Delegate to the appropriate lower-level constructor. + // Delegate to the appropriate lower-level constructor. Pass `mode` + // through to the existing-DB path so open-time recovery (pin cleanup, + // backlog drain) runs only for writer-lock-holding modes. let mut db = if main_db_exists { - Self::open_existing_with_options(vfs, kek, page_size, realm, options).await? + Self::open_existing_inner(vfs, kek, page_size, realm, options, mode).await? } else { Self::open_internal_with_options(vfs, kek, page_size, realm, options).await? }; diff --git a/src/txn/db/rekey.rs b/src/txn/db/rekey.rs index 061e9aa..127c3ff 100644 --- a/src/txn/db/rekey.rs +++ b/src/txn/db/rekey.rs @@ -97,6 +97,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: state.next_page_id, })?; let new_slot = commit_header( @@ -336,6 +337,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; @@ -396,6 +398,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; diff --git a/src/txn/db/segment.rs b/src/txn/db/segment.rs index af8a251..fb5a308 100644 --- a/src/txn/db/segment.rs +++ b/src/txn/db/segment.rs @@ -250,6 +250,7 @@ impl Db { catalog_root: catalog_root_bytes, commit_history_root_page_id: 0, commit_history_root_version: 0, + free_list_root_page_id: state.free_list_root_page_id, next_page_id: new_next, })?; diff --git a/src/txn/db/util.rs b/src/txn/db/util.rs index f13f069..43f97a9 100644 --- a/src/txn/db/util.rs +++ b/src/txn/db/util.rs @@ -5,7 +5,6 @@ use std::sync::Arc; use crate::btree::BTree; -use crate::catalog::codec::Catalog; use crate::crypto::kdf::{derive_hk, derive_mk}; use crate::errors::PagedbError; use crate::pager::Pager; @@ -101,10 +100,24 @@ pub(super) fn next_lease_id() -> u64 { ts ^ (seq.wrapping_mul(0x9e37_79b9_7f4a_7c15)) } -/// Scan the catalog for durable reader-pin rows that are either stale (written -/// by the current PID from a previous process incarnation) or expired by wall -/// clock. Delete all such rows in a single bulk catalog commit. Called at -/// writer-open time to recover from reader crashes. +/// Delete **every** durable reader-pin row in the catalog in a single bulk +/// catalog commit. Called at writer-open time, and only for handle modes that +/// hold the exclusive writer lock (Standalone / Follower). +/// +/// Why deleting *all* pins is correct — not just own-PID or wall-clock-expired +/// ones: durable pin rows are written exclusively by a Standalone writer's +/// `begin_read` (other modes track readers in memory only). A Standalone / +/// Follower handle holds the exclusive writer lock for its entire lifetime, and +/// this cleanup runs while that lock is held at open, *before* any reader has +/// been handed out on the new handle. Therefore no live reader of the current +/// incarnation can own a pin row yet, and no other process can hold the writer +/// lock concurrently. Every pin row present at this moment is a leftover from a +/// prior, now-dead incarnation and is safe to remove. +/// +/// This supersedes the earlier own-PID-or-expired heuristic, which could not +/// remove a pin left by a *crashed* process running under a different PID until +/// its 30 s lease expired — and which, combined with `min_durable_reader_commit`, +/// let such a pin permanently block deferred-free drain and compaction. #[allow(clippy::too_many_arguments)] pub(crate) async fn cleanup_stale_reader_pins( pager: &Arc>, @@ -122,8 +135,6 @@ pub(crate) async fn cleanup_stale_reader_pins( if state.catalog_root_page_id == 0 { return Ok(()); } - let now = crate::txn::read::unix_now_seconds(); - let own_pid = std::process::id(); let tree = BTree::open( pager.clone(), realm_id, @@ -135,29 +146,10 @@ pub(crate) async fn cleanup_stale_reader_pins( let end = crate::catalog::codec::Catalog::reader_pin_range_end(); let rows = tree.collect_range(&start, &end).await?; - let stale_keys: Vec> = rows - .into_iter() - .filter_map(|(k, v)| { - // Key layout: [0x06] || pid_u32_be[4] || lease_id_u64_be[8] - if k.len() < 13 { - return Some(k); - } - let mut pid_buf = [0u8; 4]; - pid_buf.copy_from_slice(&k[1..5]); - let row_pid = u32::from_be_bytes(pid_buf); - // Own-PID rows from prior incarnation (crash without cleanup). - if row_pid == own_pid { - return Some(k); - } - // Expired rows. - if let Ok(pv) = Catalog::decode_reader_pin(&v) { - if pv.expires_unix_seconds < now { - return Some(k); - } - } - None - }) - .collect(); + // Every pin row present at writer open is a prior-incarnation leftover + // (see the function doc for the exclusive-writer-lock argument). Remove + // all of them. + let stale_keys: Vec> = rows.into_iter().map(|(k, _v)| k).collect(); if stale_keys.is_empty() { return Ok(()); @@ -193,7 +185,7 @@ pub(crate) async fn cleanup_stale_reader_pins( active_root_txn_id: state.latest_commit_id, counter_anchor, commit_id: CommitId(new_commit_id), - free_list_root: [0u8; 16], + free_list_root: super::core::encode_free_list_root(state.free_list_root_page_id), catalog_root: catalog_root_bytes, apply_journal_root_page_id: 0, apply_journal_root_version: 0, @@ -221,3 +213,104 @@ pub(crate) async fn cleanup_stale_reader_pins( state.seq = new_seq; Ok(()) } + +#[cfg(test)] +mod tests { + use std::sync::atomic::Ordering; + + use super::super::core::Db; + use crate::btree::BTree; + use crate::catalog::codec::Catalog; + use crate::txn::read::{insert_pin_row, make_pin_handle}; + use crate::vfs::memory::MemVfs; + use crate::{OpenOptions, RealmId}; + + /// Count the durable reader-pin rows currently in the catalog. + async fn pin_row_count(db: &Db) -> usize { + let (root, next) = { + let w = db.writer.lock().await; + (w.catalog_root_page_id, w.next_page_id) + }; + if root == 0 { + return 0; + } + let tree = BTree::open(db.pager.clone(), db.realm_id, root, next, db.page_size); + let start = Catalog::reader_pin_range_start(); + let end = Catalog::reader_pin_range_end(); + tree.collect_range(&start, &end).await.unwrap().len() + } + + /// A durable reader-pin row written by a *crashed* process running under a + /// different PID, with a lease far in the future, must be cleared the next + /// time a writer opens the store — otherwise it permanently pins an old + /// commit and blocks deferred-free drain / compaction. The old own-PID-or- + /// expired heuristic left such a row in place until its lease expired. + #[tokio::test(flavor = "current_thread")] + async fn open_clears_foreign_pid_reader_pins() { + let vfs = MemVfs::new(); + let kek = [3u8; 32]; + let realm = RealmId::new([9u8; 16]); + + // Create the store and a catalog root. + { + let db = Db::open(vfs.clone(), kek, 4096, realm, OpenOptions::default()) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + w.put(b"seed", b"v").await.unwrap(); + w.commit().await.unwrap(); + } + + // Inject a durable pin row owned by a foreign PID with a lease + // ~28 hours in the future (definitely not expired). + { + let mut state = db.writer.lock().await; + let foreign_pid = std::process::id().wrapping_add(0x5151); + let pin = make_pin_handle( + db.pager.clone(), + db.realm_id, + db.page_size, + db.main_db_path.clone(), + db.vfs.clone(), + db.hk.read().clone(), + db.mk_epoch.load(Ordering::SeqCst), + db.cipher_id, + db.file_id, + db.kek_salt, + db.latest_commit.load(Ordering::SeqCst), + db.snapshot.clone(), + foreign_pid, + 0x777, + 100_000, + ); + let (cid, root, cat) = ( + state.latest_commit_id, + state.root_page_id, + state.catalog_root_page_id, + ); + insert_pin_row(&pin, &mut state, cid, root, cat) + .await + .unwrap(); + } + + assert_eq!( + pin_row_count(&db).await, + 1, + "foreign-PID pin row should be present before reopen" + ); + // Drop the handle (releases the writer lock) without cleaning up — + // simulating the foreign writer's crash. + } + + // Reopen as a writer: open-time cleanup must remove the foreign pin. + let db2 = Db::open(vfs.clone(), kek, 4096, realm, OpenOptions::default()) + .await + .unwrap(); + assert_eq!( + pin_row_count(&db2).await, + 0, + "open must clear every durable reader-pin row left by a prior incarnation" + ); + } +} diff --git a/src/txn/read.rs b/src/txn/read.rs index 2b156a0..2b15236 100644 --- a/src/txn/read.rs +++ b/src/txn/read.rs @@ -411,7 +411,9 @@ fn make_header_fields( active_root_txn_id: state.latest_commit_id, counter_anchor, commit_id: crate::CommitId(new_commit_id), - free_list_root: [0u8; 16], + // Preserve the durable free-list head across pin-row commits, or every + // begin_read would wipe the free list. + free_list_root: crate::txn::db::encode_free_list_root(state.free_list_root_page_id), catalog_root: catalog_root_bytes, apply_journal_root_page_id: 0, apply_journal_root_version: 0, diff --git a/src/txn/write.rs b/src/txn/write.rs deleted file mode 100644 index 554776a..0000000 --- a/src/txn/write.rs +++ /dev/null @@ -1,896 +0,0 @@ -//! `WriteTxn` — exclusive write session backed by a `CoW` B+ tree. On commit, -//! flushes dirty pages, writes the new A/B header, and advances the -//! visibility commit id. - -use std::sync::atomic::Ordering; - -use tracing; - -use tokio::sync::MutexGuard; - -use crate::btree::BTree; -use crate::catalog::codec::{Catalog, RealmQuotas, SegmentMeta}; -use crate::crypto::aad::{Aad, AadFields}; -use crate::crypto::cipher::Cipher; -use crate::crypto::kdf::derive_spill_key; -use crate::crypto::nonce::Nonce; -use crate::errors::{PagedbError, QuotaKind}; -use crate::pager::header::commit_header; -use crate::pager::structural_header::MainDbHeaderFields; -use crate::vfs::types::OpenMode; -use crate::vfs::{Vfs, VfsFile}; -use crate::{CommitId, RealmId, Result}; - -use super::db::{CommitHistoryMeta, Db, PendingTombstone, WriterState}; - -/// Opaque handle returned by [`SpillScope::append`]. Pass to -/// [`SpillScope::read`] to decrypt and retrieve the bytes. -#[derive(Debug, Clone, Copy)] -pub struct ScratchOffset(u64); - -/// Metadata for one ciphertext chunk in the per-txn spill scratch file. -/// Stored in memory only; the tmp file is discarded at commit/abort. -#[derive(Clone)] -pub(crate) struct SpillSegmentMeta { - /// Byte offset of this chunk (ciphertext body start) in the tmp file. - pub offset: u64, - /// Length of the original plaintext in bytes. - pub plaintext_len: u32, - /// Length of ciphertext body (without the 16-byte tag) in bytes. - /// Total on-disk size for this chunk = `ciphertext_len + 16`. - pub ciphertext_len: u32, - /// The 12-byte nonce used to encrypt this chunk, stored verbatim so we - /// can reconstruct a `Nonce` on read without an additional lookup. - pub nonce_bytes: [u8; 12], -} - -/// A borrowed view into an active `WriteTxn` that offers AEAD-encrypted spill -/// storage in a per-transaction tmp file. -/// -/// The tmp file lives at `tmp/scratch-`. It is created lazily on the -/// first `append` call and removed at `commit` or `abort` (best-effort). -/// -/// Nonce scheme: `txn_seq_le6 ‖ segment_index_le6`. -/// - First 6 bytes: the low 6 bytes of `txn_seq` in little-endian order. -/// - Last 6 bytes: the low 6 bytes of the per-append segment index in -/// little-endian order. -/// -/// This guarantees uniqueness across all appends within a txn and across -/// independent txns (different `txn_seq`), without requiring a durable -/// nonce anchor (the tmp file is discarded before any subsequent txn can -/// reuse the same `txn_seq`). -pub struct SpillScope<'scope, 'db, V: Vfs + Clone> { - txn: &'scope mut WriteTxn<'db, V>, -} - -impl SpillScope<'_, '_, V> { - /// Encrypt `bytes` under the per-txn spill key and append the ciphertext - /// plus 16-byte AEAD tag to the tmp file. Returns a `ScratchOffset` handle - /// usable with [`read`](Self::read). - /// - /// Returns `PagedbError::Quota { kind: QuotaKind::ScratchPages, … }` when - /// the cumulative ciphertext bytes (body + tag) would exceed the - /// `OpenOptions::scratch_bytes` budget. - pub async fn append(&mut self, bytes: &[u8]) -> Result { - let limit = self.txn.db.options.scratch_bytes as u64; - let segment_index = self.txn.spill_segments.len() as u64; - - // Extract immutable db fields before taking a mutable borrow for the - // cipher. Rust cannot prove these are disjoint borrows through the - // &mut reference, so we copy them out first. - let mk_epoch = self - .txn - .db - .mk_epoch - .load(std::sync::atomic::Ordering::SeqCst); - let realm_id = self.txn.db.realm_id; - let file_id = self.txn.db.file_id; - - // Build the per-append nonce: txn_seq_le6 || segment_index_le6. - let nonce = self.txn.next_spill_nonce(segment_index); - - // Derive the spill cipher lazily (mutably borrows txn, then releases). - self.txn.ensure_spill_cipher()?; - let cipher_id_byte = self - .txn - .spill_cipher - .as_ref() - .expect("just derived") - .id() - .as_byte(); - - // Build AAD: binds cipher_id, a spill sentinel page_kind (0xFE), - // mk_epoch, this segment's index (as page_id), realm_id, and file_id. - let aad = Aad::from_fields(AadFields { - cipher_id: cipher_id_byte, - page_kind: 0xFE, - mk_epoch, - page_id: segment_index, - realm_id, - segment_id: file_id, - }); - - let mut body = bytes.to_vec(); - let tag = self - .txn - .spill_cipher - .as_ref() - .expect("derived above") - .encrypt(&nonce, &aad, &mut body)?; - - // ciphertext body + 16-byte tag. - let pers_len = body.len() as u64 + 16; - let new_total = self.txn.spill_bytes_used.saturating_add(pers_len); - if new_total > limit { - return Err(PagedbError::quota( - self.txn.db.realm_id, - QuotaKind::ScratchPages, - new_total, - limit, - )); - } - - // Lazy: create tmp dir and file on first append. - let path = self.txn.ensure_spill_path().await?; - - let mut file = self.txn.db.vfs.open(&path, OpenMode::CreateOrOpen).await?; - let body_offset = self.txn.spill_bytes_used; - let tag_offset = body_offset + body.len() as u64; - file.write_at(body_offset, &body).await?; - file.write_at(tag_offset, &tag).await?; - file.sync().await?; - - let plaintext_len = u32::try_from(bytes.len()).map_err(|_| PagedbError::PayloadTooLarge)?; - let ciphertext_len = u32::try_from(body.len()).map_err(|_| PagedbError::PayloadTooLarge)?; - - self.txn.spill_segments.push(SpillSegmentMeta { - offset: body_offset, - plaintext_len, - ciphertext_len, - nonce_bytes: *nonce.as_bytes(), - }); - self.txn.spill_bytes_used = new_total; - self.txn - .db - .spill_bytes_in_use - .store(new_total, std::sync::atomic::Ordering::Relaxed); - - Ok(ScratchOffset(segment_index)) - } - - /// Decrypt and return the bytes previously written via - /// [`append`](Self::append) at `handle`. - pub async fn read(&self, handle: ScratchOffset) -> Result> { - let idx = usize::try_from(handle.0).map_err(|_| PagedbError::NotFound)?; - let meta = self - .txn - .spill_segments - .get(idx) - .ok_or(PagedbError::NotFound)? - .clone(); - let path = self.txn.spill_path.as_ref().ok_or(PagedbError::NotFound)?; - - let file = self.txn.db.vfs.open(path, OpenMode::Read).await?; - - let body_len = meta.ciphertext_len as usize; - let mut body = vec![0u8; body_len]; - let mut tag = [0u8; 16]; - file.read_at(meta.offset, &mut body).await?; - file.read_at(meta.offset + body_len as u64, &mut tag) - .await?; - - let cipher = self.txn.spill_cipher_readonly()?; - let nonce = Nonce::from_bytes(meta.nonce_bytes); - let aad = Aad::from_fields(AadFields { - cipher_id: cipher.id().as_byte(), - page_kind: 0xFE, - mk_epoch: self - .txn - .db - .mk_epoch - .load(std::sync::atomic::Ordering::SeqCst), - page_id: handle.0, - realm_id: self.txn.db.realm_id, - segment_id: self.txn.db.file_id, - }); - - cipher.decrypt(&nonce, &aad, &mut body, &tag)?; - body.truncate(meta.plaintext_len as usize); - Ok(body) - } -} - -/// A handle to a named durable monotonic counter, scoped to a `WriteTxn`. -/// -/// The counter starts at `0` if it has never been written. Values are stored -/// as 8-byte little-endian `u64` catalog rows with key prefix `0x02`. -/// -/// Monotonicity guarantee: `set(v)` returns `PagedbError::Aborted` when -/// `v < current`; `increment_by` can never produce a value smaller than the -/// current one. Values only survive once the enclosing `WriteTxn::commit` -/// succeeds. -pub struct CounterRef<'a, V: Vfs + Clone> { - catalog_tree: &'a mut BTree, - main_tree: &'a mut BTree, - key: Vec, -} - -impl CounterRef<'_, V> { - /// Return the current counter value, or `0` if no row exists yet. - pub async fn get(&self) -> Result { - match self.catalog_tree.get(&self.key).await? { - Some(v) => Catalog::decode_counter(&v), - None => Ok(0), - } - } - - /// Set the counter to `value`. Returns `PagedbError::Aborted` when - /// `value < current` (strict monotonicity). - pub async fn set(&mut self, value: u64) -> Result<()> { - let current = self.get().await?; - if value < current { - return Err(PagedbError::Aborted); - } - Self::sync_to(self.main_tree, self.catalog_tree); - let bytes = Catalog::encode_counter(value); - self.catalog_tree.put(&self.key, &bytes).await?; - Self::sync_from(self.catalog_tree, self.main_tree); - Ok(()) - } - - /// Increment the counter by `delta` and return the new value. Returns - /// `PagedbError::NonceCounterExhausted` on `u64` overflow. - pub async fn increment_by(&mut self, delta: u64) -> Result { - let current = self.get().await?; - let next = current - .checked_add(delta) - .ok_or(PagedbError::NonceCounterExhausted)?; - Self::sync_to(self.main_tree, self.catalog_tree); - let bytes = Catalog::encode_counter(next); - self.catalog_tree.put(&self.key, &bytes).await?; - Self::sync_from(self.catalog_tree, self.main_tree); - Ok(next) - } - - /// Advance the catalog allocator cursor to be at least as high as the - /// main tree's before a catalog write. - fn sync_to(main: &BTree, catalog: &mut BTree) { - let shared = main.next_page_id().max(catalog.next_page_id()); - catalog.set_next_page_id(shared); - } - - /// After a catalog write, propagate any advances back to the main tree. - fn sync_from(catalog: &BTree, main: &mut BTree) { - let c = catalog.next_page_id(); - if main.next_page_id() < c { - main.set_next_page_id(c); - } - } -} - -/// Deferred filesystem operation applied after the A/B header is durable. -pub(crate) enum SegmentSideEffect { - Promote { segment_id: [u8; 16] }, - Tombstone { segment_id: [u8; 16] }, -} - -/// An exclusive write transaction. At most one `WriteTxn` exists per `Db` at -/// any time — the writer mutex enforces this. Either `commit` or `abort` must -/// be called; if the `WriteTxn` is dropped without either, dirty pages are -/// silently discarded (equivalent to abort). -pub struct WriteTxn<'db, V: Vfs + Clone> { - db: &'db Db, - guard: MutexGuard<'db, WriterState>, - btree: BTree, - catalog_tree: BTree, - pending_segments: Vec, - committed_or_aborted: bool, - /// Monotonic per-txn sequence number; assigned at `begin` from - /// `Db::txn_seq.fetch_add(1, Relaxed) + 1`. The first `WriteTxn` on a - /// fresh Db gets `txn_seq == 1`, making `tmp/scratch-1` predictable in - /// tests. - pub(crate) txn_seq: u64, - /// Lazily derived spill cipher. `None` until the first `spill_scope` append. - pub(crate) spill_cipher: Option, - /// Lazily created path to the per-txn spill tmp file (`tmp/scratch-`). - pub(crate) spill_path: Option, - /// Cumulative bytes (ciphertext body + tag) written to the spill file. - pub(crate) spill_bytes_used: u64, - /// Per-append metadata used by `SpillScope::read` to reconstruct AAD/nonce. - pub(crate) spill_segments: Vec, -} - -impl<'db, V: Vfs + Clone> WriteTxn<'db, V> { - pub(crate) async fn begin(db: &'db Db) -> Result> { - let guard = db.writer.lock().await; - // If any reader is pinned, freed pages from prior snapshots must not be - // recycled within this txn. Set the reuse threshold to next_page_id so - // that only pages allocated in this session (and then freed within it) - // can be recycled. - let reuse_threshold = { - let readers = db.tracked_readers.lock(); - if readers.is_empty() { - 0 - } else { - guard.next_page_id - } - }; - let mut btree = BTree::open( - db.pager.clone(), - db.realm_id, - guard.root_page_id, - guard.next_page_id, - db.page_size, - ); - btree.set_reuse_threshold(reuse_threshold); - btree.set_free_page_cache(db.free_page_cache.clone()); - let mut catalog_tree = BTree::open( - db.pager.clone(), - db.realm_id, - guard.catalog_root_page_id, - guard.next_page_id, - db.page_size, - ); - catalog_tree.set_reuse_threshold(reuse_threshold); - catalog_tree.set_free_page_cache(db.free_page_cache.clone()); - // Assign a txn_seq starting from 1: fetch_add returns the old value (0 - // for the first call), so we add 1 to produce 1-based ids. - let txn_seq = db.txn_seq.fetch_add(1, Ordering::Relaxed) + 1; - Ok(Self { - db, - guard, - btree, - catalog_tree, - pending_segments: Vec::new(), - committed_or_aborted: false, - txn_seq, - spill_cipher: None, - spill_path: None, - spill_bytes_used: 0, - spill_segments: Vec::new(), - }) - } - - /// Return a [`SpillScope`] that writes AEAD-encrypted bytes to a per-txn - /// tmp file within the `scratch_bytes` budget from `OpenOptions`. - pub fn spill_scope(&mut self) -> SpillScope<'_, 'db, V> { - SpillScope { txn: self } - } - - /// Lazily derive and cache the per-txn spill cipher (AES-256-GCM keyed - /// with a spill-specific HKDF derivation from the master key). - pub(crate) fn ensure_spill_cipher(&mut self) -> Result<&Cipher> { - if self.spill_cipher.is_none() { - let pager_mk = self.db.pager.mk(); - let key = derive_spill_key(&pager_mk, &self.db.file_id, self.txn_seq)?; - self.spill_cipher = Some(Cipher::new_aes_gcm(&key)); - } - Ok(self.spill_cipher.as_ref().expect("just set")) - } - - /// Return a reference to the cached spill cipher for read paths (does NOT - /// lazily create — callers must have already called `ensure_spill_cipher` - /// or `append` at least once, which guarantees the cipher exists). - pub(crate) fn spill_cipher_readonly(&self) -> Result<&Cipher> { - self.spill_cipher.as_ref().ok_or(PagedbError::NotFound) - } - - /// Lazily create the `tmp/` directory and return the spill file path. - pub(crate) async fn ensure_spill_path(&mut self) -> Result { - if self.spill_path.is_none() { - self.db.vfs.mkdir_all("tmp").await?; - let path = format!("tmp/scratch-{}", self.txn_seq); - self.spill_path = Some(path); - } - Ok(self.spill_path.clone().expect("just set")) - } - - /// Build the per-append nonce deterministically from `(txn_seq, segment_index)`. - /// - /// Layout: `txn_seq_le6 ‖ segment_index_le6` (6 + 6 = 12 bytes). - /// Uniqueness: nonces are unique per-txn (distinct `segment_index`) and - /// across txns (distinct `txn_seq` in first 6 bytes). No durable anchor - /// is needed because the tmp file is discarded before any key reuse - /// could matter. - pub(crate) fn next_spill_nonce(&self, segment_index: u64) -> Nonce { - let mut bytes = [0u8; 12]; - let seq_le = self.txn_seq.to_le_bytes(); - let idx_le = segment_index.to_le_bytes(); - bytes[..6].copy_from_slice(&seq_le[..6]); - bytes[6..].copy_from_slice(&idx_le[..6]); - Nonce::from_bytes(bytes) - } - - /// Remove the spill tmp file if it was created. Errors are swallowed - /// (best-effort cleanup); the file is transient anyway. - pub(crate) async fn cleanup_spill_async(&mut self) { - if let Some(path) = self.spill_path.take() { - let _ = self.db.vfs.remove(&path).await; - } - } - - pub async fn get(&self, key: &[u8]) -> Result>> { - self.btree.get(key).await - } - - pub async fn put(&mut self, key: &[u8], value: &[u8]) -> Result<()> { - self.btree.put(key, value).await - } - - /// Append a key-value pair under the monotonic-key invariant. - /// - /// Subsequent calls within the same `WriteTxn` skip the - /// `path_to_leaf_for_key` descent by reusing the cached rightmost - /// path; splits and explicitly-invalidating operations (regular `put`, - /// `delete`) force a re-descent on the next call. - /// - /// Intended for op-logs, time-series indexes, FTS posting-list builds - /// — any workload where the embedder can guarantee monotonically - /// increasing keys. - /// - /// # Errors - /// - /// Returns [`PagedbError::AppendNotMonotonic`] if `key` is not - /// strictly greater than the previously-appended key in this txn. - pub async fn put_append(&mut self, key: &[u8], value: &[u8]) -> Result<()> { - self.btree.put_append(key, value).await - } - - pub async fn delete(&mut self, key: &[u8]) -> Result { - self.btree.delete(key).await - } - - pub async fn put_batch(&mut self, sorted: Vec<(Vec, Vec)>) -> Result<()> { - self.btree.put_batch(sorted).await - } - - pub async fn delete_batch(&mut self, sorted: Vec>) -> Result<()> { - self.btree.delete_batch(sorted).await - } - - pub async fn delete_range(&mut self, start: &[u8], end: &[u8]) -> Result { - self.btree.delete_range(start, end).await - } - - /// Return a `CounterRef` scoped to this transaction for the named counter. - /// - /// The returned handle borrows `self` mutably for its lifetime. Use the - /// counter, drop the handle, then continue with other transaction - /// operations. The name must be at most `MAX_SEGMENT_NAME_LEN` bytes - /// (`PagedbError::NameTooLong` otherwise). - pub fn counter<'tx>(&'tx mut self, name: &str) -> Result> { - let key = Catalog::counter_key(name.as_bytes())?; - Ok(CounterRef { - catalog_tree: &mut self.catalog_tree, - main_tree: &mut self.btree, - key, - }) - } - - /// Register a segment under `name` in the catalog and schedule promotion - /// of its staging file to the live path on commit. - pub async fn link_segment(&mut self, name: &str, meta: &SegmentMeta) -> Result<()> { - let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; - if self.catalog_tree.get(&key).await?.is_some() { - return Err(PagedbError::AlreadyLinked); - } - self.enforce_segment_bytes_quota(meta.realm_id, meta.total_bytes, 0) - .await?; - let value = Catalog::encode_segment_meta(meta); - self.sync_allocator_to_catalog(); - self.catalog_tree.put(&key, &value).await?; - self.sync_allocator_from_catalog(); - self.pending_segments.push(SegmentSideEffect::Promote { - segment_id: meta.segment_id, - }); - Ok(()) - } - - /// Remove the catalog row for `name` and schedule a tombstone rename on commit. - pub async fn unlink_segment(&mut self, name: &str) -> Result<()> { - let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; - let value = self - .catalog_tree - .get(&key) - .await? - .ok_or(PagedbError::NotLinked)?; - let meta = Catalog::decode_segment_meta(&value)?; - self.sync_allocator_to_catalog(); - let removed = self.catalog_tree.delete(&key).await?; - self.sync_allocator_from_catalog(); - if !removed { - return Err(PagedbError::NotLinked); - } - self.pending_segments.push(SegmentSideEffect::Tombstone { - segment_id: meta.segment_id, - }); - Ok(()) - } - - /// Atomically swap the segment recorded under `name`: tombstone the old - /// segment id and promote `new_meta`'s staging file on commit. - pub async fn replace_segment(&mut self, name: &str, new_meta: &SegmentMeta) -> Result<()> { - let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; - let existing = self - .catalog_tree - .get(&key) - .await? - .ok_or(PagedbError::NotLinked)?; - let old_meta = Catalog::decode_segment_meta(&existing)?; - self.enforce_segment_bytes_quota( - new_meta.realm_id, - new_meta.total_bytes, - old_meta.total_bytes, - ) - .await?; - let value = Catalog::encode_segment_meta(new_meta); - self.sync_allocator_to_catalog(); - self.catalog_tree.put(&key, &value).await?; - self.sync_allocator_from_catalog(); - self.pending_segments.push(SegmentSideEffect::Tombstone { - segment_id: old_meta.segment_id, - }); - self.pending_segments.push(SegmentSideEffect::Promote { - segment_id: new_meta.segment_id, - }); - Ok(()) - } - - /// Check if the realm's committed segment bytes plus `new_bytes` minus - /// `delta_remove_bytes` would exceed the configured cap. Returns `Ok(())` - /// when no cap is set or the projected total is within the limit. - async fn enforce_segment_bytes_quota( - &self, - realm: RealmId, - new_bytes: u64, - delta_remove_bytes: u64, - ) -> Result<()> { - let quota_key = Catalog::quota_key(realm); - let quotas = match self.catalog_tree.get("a_key).await? { - Some(v) => Catalog::decode_realm_quotas(&v)?, - None => RealmQuotas::default(), - }; - let Some(limit) = quotas.max_segment_bytes else { - return Ok(()); - }; - // Scan all catalog segment rows for this realm to sum committed bytes. - let mut prefix = Vec::with_capacity(17); - prefix.push(0x01u8); // CatalogRowKind::Segment - prefix.extend_from_slice(&realm.0); - let mut end = prefix.clone(); - end.push(0xFF); - let rows = self.catalog_tree.collect_range(&prefix, &end).await?; - let mut committed: u64 = 0; - for (_, v) in rows { - let meta = Catalog::decode_segment_meta(&v)?; - committed = committed.saturating_add(meta.total_bytes); - } - let after_remove = committed.saturating_sub(delta_remove_bytes); - let projected = after_remove.saturating_add(new_bytes); - if projected > limit { - return Err(PagedbError::quota( - realm, - QuotaKind::SegmentBytes, - projected, - limit, - )); - } - Ok(()) - } - - /// Flush dirty pages, write the A/B header, apply pending segment side - /// effects, and publish the new root to readers. Returns the assigned - /// `CommitId`. - #[allow(clippy::too_many_lines)] - pub async fn commit(mut self) -> Result { - let _span = tracing::debug_span!("txn.commit"); - // Note: span is not entered via `.entered()` to keep this async fn's - // future `Send`. Use `tracing::instrument` or enter in sync sections only. - // Compute commit_id early so we can tag deferred-free entries before - // the catalog flush. - let new_commit_id = self.guard.latest_commit_id + 1; - - // Pages freed in this commit that, on the fast-free path, will be - // pushed to the `Db`'s shared free-page cache *after* the header - // swap succeeds. Lifted out of the block below so the post-commit - // handoff can see it. - let mut fast_freed_for_cache: Vec = Vec::new(); - // Collect freed pages from both trees and add them to the - // deferred-free queue in the catalog (tagged with new_commit_id). - // This keeps the free-list persistent across reopen. - { - let freed_from_main = self.btree.drain_freed(); - let freed_from_catalog = self.catalog_tree.drain_freed(); - let all_freed: Vec = freed_from_main - .into_iter() - .chain(freed_from_catalog) - // Skip reserved pages (0..=3) — they must never enter the free-list. - .filter(|&pid| pid >= 4) - .collect(); - - // Track the deferred-free pair count for the stall-policy check. - // Populated below from `free_pages_deferred_batch`'s return value - // when we wrote pairs this commit; otherwise read once on demand. - let mut deferred_count: Option = None; - // Opt-in fast path: when the embedder has set - // `skip_freelist_persistence_when_no_readers` AND no readers are - // pinned at this commit, skip the catalog-tree CoW for the - // deferred-free row. Instead of orphaning the freed pages we - // route them into the `Db`'s shared `free_page_cache` further - // down (after the header swap is durable, so a mid-commit crash - // can't surface them prematurely). The next writer txn's - // allocator pops from there, keeping the file size bounded. - // This is a pagedb extension beyond the architecture spec, - // documented on - // `OpenOptions::skip_freelist_persistence_when_no_readers`. The - // default behavior (option `false`) is spec-conformant: every - // freed page is persisted to the deferred-free queue. - let skip_persist = self.db.options.skip_freelist_persistence_when_no_readers - && self.db.tracked_readers.lock().is_empty(); - // Stash the list for post-commit handoff to the shared cache. - // Cleared if `skip_persist` is false (pages go to the catalog - // queue instead) or empty. - if skip_persist { - fast_freed_for_cache = all_freed.clone(); - } - if !all_freed.is_empty() && !skip_persist { - self.sync_allocator_to_catalog(); - // Batch all freed pages into a single deferred-free put to - // avoid repeated CoW on the deferred-free row. - deferred_count = Some( - crate::compaction::freelist::free_pages_deferred_batch( - &mut self.catalog_tree, - new_commit_id, - &all_freed, - ) - .await?, - ); - self.sync_allocator_from_catalog(); - } - - // Evaluate the reader stall policy against the current deferred-free - // queue depth. On Reject or all-non-abortable AbortOldest this returns - // an error and the commit is aborted by the `?` propagation. - { - let count = if let Some(c) = deferred_count { - c - } else { - let dk = crate::catalog::codec::Catalog::deferred_free_key(); - match self.catalog_tree.get(&dk).await? { - Some(bytes) => (bytes.len() as u64) / 16, - None => 0, - } - }; - self.db.evaluate_stall_policy(count)?; - } - } - - // Materialize each tree's dirty-leaf cache into the pager WITHOUT - // fsyncing per tree. All three trees (main, catalog, history) share - // the same file/realm, so their dirty pages live in one pager dirty - // set — a single `pager.flush_main` below batches them into one - // fsync. `materialize_dirty` allocates new pages, so re-sync the - // shared allocator between trees. - self.btree.materialize_dirty().await?; - self.sync_allocator_to_catalog(); - self.catalog_tree.materialize_dirty().await?; - self.sync_allocator_from_catalog(); - let new_root = self.btree.root_page_id(); - let new_catalog_root = self.catalog_tree.root_page_id(); - let new_next_after_trees = self - .btree - .next_page_id() - .max(self.catalog_tree.next_page_id()); - - // Stage the guard's next_page_id so the history tree allocates from - // above the main/catalog pages. - self.guard.next_page_id = new_next_after_trees; - - // Timestamp for Age-based retention. - let unix_seconds = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - - let history_meta = CommitHistoryMeta { - active_root_page_id: new_root, - catalog_root_page_id: new_catalog_root, - free_list_root_page_id: 0, - next_page_id: new_next_after_trees, - unix_seconds, - }; - // Skip the entire commit-history tree maintenance when the policy is - // Disabled. Saves the per-commit history-tree CoW + flush chain. - if !matches!( - self.db.options.commit_history_retain, - crate::options::RetainPolicy::Disabled - ) { - self.db - .write_commit_history_entry(&mut self.guard, new_commit_id, history_meta) - .await?; - } - - // Single pager fsync for all three trees' dirty pages. Replaces the - // per-tree fsyncs that used to come from `btree.flush`, - // `catalog_tree.flush`, and `hist_tree.flush` — collapses three - // fsyncs into one before the header write. - self.db.pager.flush_main(self.db.realm_id).await?; - - let new_next = self.guard.next_page_id; - let new_seq = self.guard.seq + 1; - let counter_anchor = self.db.pager.pending_anchor(); - - let mut catalog_root_bytes = [0u8; 16]; - catalog_root_bytes[..8].copy_from_slice(&new_catalog_root.to_le_bytes()); - catalog_root_bytes[8..].copy_from_slice(&new_commit_id.to_le_bytes()); - - let (policy_tag, policy_value) = - encode_retain_policy(&self.db.options.commit_history_retain); - - let fields = MainDbHeaderFields { - format_version: 1, - cipher_id: self.db.cipher_id.as_byte(), - page_size_log2: page_size_log2(self.db.page_size)?, - flags: 0, - file_id: self.db.file_id, - kek_salt: self.db.kek_salt, - mk_epoch: self.db.mk_epoch.load(std::sync::atomic::Ordering::SeqCst), - seq: new_seq, - active_root_page_id: new_root, - active_root_txn_id: new_commit_id, - counter_anchor, - commit_id: CommitId(new_commit_id), - free_list_root: [0; 16], - catalog_root: catalog_root_bytes, - apply_journal_root_page_id: 0, - apply_journal_root_version: 0, - commit_history_root_page_id: self.guard.commit_history_root_page_id, - commit_history_root_version: self.guard.commit_history_root_version, - restore_mode: 0, - next_page_id: new_next, - commit_retain_policy_tag: policy_tag, - commit_retain_policy_value: policy_value, - }; - - let hk_clone = { self.db.hk.read().clone() }; - let new_slot = commit_header( - &*self.db.vfs, - &self.db.main_db_path, - &hk_clone, - &fields, - self.guard.active_slot, - self.db.page_size, - ) - .await?; - self.db.pager.commit_anchor(counter_anchor)?; - - // Apply pending segment side effects after the header is durable. - if !self.pending_segments.is_empty() { - self.db.vfs.mkdir_all("seg").await?; - self.db.vfs.sync_dir("seg").await?; - self.db.vfs.mkdir_all("seg/.tombstone").await?; - self.db.vfs.sync_dir("seg/.tombstone").await?; - for effect in &self.pending_segments { - match effect { - SegmentSideEffect::Promote { segment_id } => { - let staging = crate::segment::writer::staging_path(segment_id); - let live = crate::segment::writer::live_path(segment_id); - self.db.vfs.rename(&staging, &live).await?; - } - SegmentSideEffect::Tombstone { segment_id } => { - if self.db.segment_id_is_reader_pinned(*segment_id).await? { - self.db.pending_tombstones.lock().push(PendingTombstone { - segment_id: *segment_id, - commit_id: new_commit_id, - }); - } else { - let live = crate::segment::writer::live_path(segment_id); - let tomb = format!( - "seg/.tombstone/{}.{}", - crate::hex::to_hex_lower(segment_id), - new_commit_id, - ); - self.db.vfs.rename(&live, &tomb).await?; - } - } - } - } - self.db.vfs.sync_dir("seg").await?; - self.db.vfs.sync_dir("seg/.tombstone").await?; - } - - self.guard.root_page_id = new_root; - self.guard.next_page_id = new_next; - self.guard.active_slot = new_slot; - self.guard.seq = new_seq; - self.guard.latest_commit_id = new_commit_id; - self.guard.catalog_root_page_id = new_catalog_root; - self.guard.catalog_root_txn_id = new_commit_id; - // commit_history_root_page_id and commit_history_root_version are - // already updated inside write_commit_history_entry. - self.db.latest_commit.store(new_commit_id, Ordering::SeqCst); - self.committed_or_aborted = true; - - // Post-commit handoff to the shared free-page cache. Only happens - // on the opt-in fast-free path (`fast_freed_for_cache` is empty - // otherwise). Deferred to here so a mid-commit failure aborts - // before the pages become "available for reuse" — the still-active - // previous root may still reference them. - if !fast_freed_for_cache.is_empty() { - let mut cache = self.db.free_page_cache.lock(); - cache.extend(fast_freed_for_cache.iter().copied()); - } - - // Remove the spill tmp file (best-effort; error is non-fatal). - self.cleanup_spill_async().await; - - self.db - .spill_bytes_in_use - .store(0, std::sync::atomic::Ordering::Relaxed); - tracing::debug!( - name = "txn.commit", - commit_id = new_commit_id, - "write transaction committed" - ); - Ok(CommitId(new_commit_id)) - } - - /// Discard all in-flight dirty pages without writing anything durable. - /// The spill tmp file (if created) is removed before returning (best-effort; - /// errors are ignored). - pub async fn abort(mut self) { - tracing::debug!(name = "txn.abort", "write transaction aborted"); - self.db.pager.discard_dirty_main(self.db.realm_id); - self.committed_or_aborted = true; - self.db - .spill_bytes_in_use - .store(0, std::sync::atomic::Ordering::Relaxed); - self.cleanup_spill_async().await; - } - - /// Before a catalog operation: ensure the catalog tree's allocator cursor - /// is at least as high as the main tree's, so neither tree allocates the - /// same page id. - fn sync_allocator_to_catalog(&mut self) { - let main_next = self.btree.next_page_id(); - let cat_next = self.catalog_tree.next_page_id(); - let shared = main_next.max(cat_next); - self.catalog_tree.set_next_page_id(shared); - } - - /// After a catalog operation: propagate any catalog allocation advances - /// back to the main tree so subsequent main-tree allocations stay above. - fn sync_allocator_from_catalog(&mut self) { - let cat_next = self.catalog_tree.next_page_id(); - self.btree.set_next_page_id(cat_next); - } -} - -impl Drop for WriteTxn<'_, V> { - fn drop(&mut self) { - if !self.committed_or_aborted { - self.db.pager.discard_dirty_main(self.db.realm_id); - } - } -} - -fn page_size_log2(page_size: usize) -> Result { - match page_size { - 4096 => Ok(12), - 8192 => Ok(13), - 16384 => Ok(14), - 32768 => Ok(15), - 65536 => Ok(16), - _ => Err(PagedbError::Unsupported), - } -} - -/// Encode a `RetainPolicy` into the two header fields `(tag, value)`. -/// tag 0 = Count, tag 1 = Age (seconds), tag 2 = Unbounded, tag 3 = Disabled. -fn encode_retain_policy(policy: &crate::options::RetainPolicy) -> (u8, u64) { - match policy { - crate::options::RetainPolicy::Count(n) => (0, u64::from(*n)), - crate::options::RetainPolicy::Age(d) => (1, d.as_secs()), - crate::options::RetainPolicy::Unbounded => (2, 0), - crate::options::RetainPolicy::Disabled => (3, 0), - } -} diff --git a/src/txn/write/commit.rs b/src/txn/write/commit.rs new file mode 100644 index 0000000..6bdea41 --- /dev/null +++ b/src/txn/write/commit.rs @@ -0,0 +1,294 @@ +//! `WriteTxn::commit` — flush dirty pages, rewrite the durable free-list, +//! write the A/B header, apply pending segment side effects, and publish the +//! new root to readers. +//! +//! ## Free-page reclamation +//! +//! A B+ tree commit copies-on-write every modified leaf and its spine, freeing +//! the old pages. The set of free pages lives in a durable chain rooted at the +//! header's `free_list_root` (see [`crate::pager::freelist`]) — outside the +//! catalog tree, so maintaining it adds no catalog churn and survives an +//! unclean shutdown. +//! +//! Each commit, after materializing all three trees, rebuilds the chain: +//! `(free pages before) − (pages reused this commit) + (pages freed this +//! commit) + (the old chain's own pages)`. Every entry is tagged with the +//! commit that freed it; `begin` recycles only those below the reclamation +//! floor (observable by no reader and no retained-history root). The chain is +//! committed atomically with the header swap, so no freed page is ever lost as +//! an orphan, and a page is recycled only after the commit that overwrote it is +//! durable. +//! +//! Crash-safety of the rewrite: new chain pages are drawn only from pages that +//! were *already free* before this commit (or freshly bump-allocated) — never +//! from pages this commit freed (still live under the old root until the swap) +//! nor from the old chain's own pages (must stay readable until the swap). On a +//! crash before the header swap, the old `free_list_root` and everything it +//! references are intact. + +use std::collections::HashSet; +use std::sync::atomic::Ordering; + +use crate::errors::PagedbError; +use crate::pager::freelist; +use crate::pager::header::commit_header; +use crate::pager::structural_header::MainDbHeaderFields; +use crate::vfs::Vfs; +use crate::{CommitId, Result}; + +use super::super::db::{CommitHistoryMeta, PendingTombstone, encode_free_list_root}; +use super::txn::{SegmentSideEffect, WriteTxn}; + +impl WriteTxn<'_, V> { + /// Flush dirty pages, write the A/B header, apply pending segment side + /// effects, and publish the new root to readers. Returns the assigned + /// `CommitId`. + #[allow(clippy::too_many_lines)] + pub async fn commit(mut self) -> Result { + let _span = tracing::debug_span!("txn.commit"); + // Note: span is not entered via `.entered()` to keep this async fn's + // future `Send`. Use `tracing::instrument` or enter in sync sections only. + let new_commit_id = self.guard.latest_commit_id + 1; + + // ── Materialize all trees first ────────────────────────────────────── + // Done before accounting freed pages so every copy-on-write spine free + // (which is realized during the flush, not before it) is captured. + self.btree.materialize_dirty().await?; + self.sync_allocator_to_catalog(); + self.catalog_tree.materialize_dirty().await?; + self.sync_allocator_from_catalog(); + let new_root = self.btree.root_page_id(); + let new_catalog_root = self.catalog_tree.root_page_id(); + self.guard.next_page_id = self + .btree + .next_page_id() + .max(self.catalog_tree.next_page_id()); + + // Commit-history entry (also materialized here). Its frees are never + // reader-pinned, so they fold into the free-list like any other. + let unix_seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + let history_meta = CommitHistoryMeta { + active_root_page_id: new_root, + catalog_root_page_id: new_catalog_root, + free_list_root_page_id: self.guard.free_list_root_page_id, + next_page_id: self.guard.next_page_id, + unix_seconds, + }; + let mut hist_freed: Vec = Vec::new(); + if !matches!( + self.db.options.commit_history_retain, + crate::options::RetainPolicy::Disabled + ) { + hist_freed = self + .db + .write_commit_history_entry(&mut self.guard, new_commit_id, history_meta) + .await?; + } + + // ── Account freed and reused pages ─────────────────────────────────── + // Pages freed by this commit (now live under the *old* root until the + // header swaps). Reserved pages 0..=3 are header/spare slots. + let all_freed: Vec = self + .btree + .drain_freed() + .into_iter() + .chain(self.catalog_tree.drain_freed()) + .chain(hist_freed) + .filter(|&pid| pid >= 4) + .collect(); + // Pages the allocator reused from the cache this txn — they were free + // before, now hold live committed data, so they leave the free-list. + let consumed: HashSet = std::mem::take(&mut *self.db.free_page_consumed.lock()) + .into_iter() + .collect(); + + // All free pages after this commit: the loaded chain's entries minus + // the ones reused, kept with their original freeing-commit tag. + let prior_free: Vec<(u64, u64)> = std::mem::take(&mut self.free_set_loaded) + .into_iter() + .filter(|(_, pid)| !consumed.contains(pid)) + .collect(); + let old_chain: Vec = std::mem::take(&mut self.old_chain_pages); + + // Pages safe to *host* the rewritten chain: only those below the + // reclamation floor and not reused — i.e. the remaining contents of the + // allocator cache (loaded at begin with exactly the floor-safe pages). + // A free-list entry with `cid >= floor` is a page a pinned reader still + // sees (it was freed *after* that reader's snapshot), so it must never + // be overwritten — only carried forward as an entry. + let host_candidates: Vec = std::mem::take(&mut *self.db.free_page_cache.lock()) + .into_iter() + .filter(|pid| !consumed.contains(pid)) + .collect(); + + // Assemble the new free-list: prior-free pages (kept with their original + // freeing-commit tag) plus this commit's frees and the now-superseded + // old chain pages, both tagged with this commit. + let mut entries: Vec<(u64, u64)> = prior_free; + entries.extend(all_freed.iter().map(|&pid| (new_commit_id, pid))); + entries.extend(old_chain.iter().map(|&pid| (new_commit_id, pid))); + + // The reader-stall policy fires only on entries genuinely stuck behind + // a pin — those at/above the reclamation floor. Drainable entries (below + // it) are being recycled and must not count, or an inherited-but- + // drainable backlog would spuriously abort a reader on reopen. Evaluated + // before the chain write so a reject aborts cleanly. + let stuck = entries + .iter() + .filter(|(cid, _)| *cid >= self.reclaim_floor) + .count() as u64; + self.db.evaluate_stall_policy(stuck)?; + + // Rewrite the chain, hosting it on the floor-safe pages (the cache + // remainder) and bump-allocating only if those run out. + let (new_free_list_root, new_next_page) = freelist::rewrite_chain( + &self.db.pager, + self.db.realm_id, + self.db.page_size, + entries, + host_candidates, + self.guard.next_page_id, + ) + .await?; + self.guard.next_page_id = new_next_page; + + // ── Flush + header swap ────────────────────────────────────────────── + self.db.pager.flush_main(self.db.realm_id).await?; + + let new_next = self.guard.next_page_id; + let new_seq = self.guard.seq + 1; + let counter_anchor = self.db.pager.pending_anchor(); + + let mut catalog_root_bytes = [0u8; 16]; + catalog_root_bytes[..8].copy_from_slice(&new_catalog_root.to_le_bytes()); + catalog_root_bytes[8..].copy_from_slice(&new_commit_id.to_le_bytes()); + + let (policy_tag, policy_value) = + encode_retain_policy(&self.db.options.commit_history_retain); + + let fields = MainDbHeaderFields { + format_version: 1, + cipher_id: self.db.cipher_id.as_byte(), + page_size_log2: page_size_log2(self.db.page_size)?, + flags: 0, + file_id: self.db.file_id, + kek_salt: self.db.kek_salt, + mk_epoch: self.db.mk_epoch.load(std::sync::atomic::Ordering::SeqCst), + seq: new_seq, + active_root_page_id: new_root, + active_root_txn_id: new_commit_id, + counter_anchor, + commit_id: CommitId(new_commit_id), + free_list_root: encode_free_list_root(new_free_list_root), + catalog_root: catalog_root_bytes, + apply_journal_root_page_id: 0, + apply_journal_root_version: 0, + commit_history_root_page_id: self.guard.commit_history_root_page_id, + commit_history_root_version: self.guard.commit_history_root_version, + restore_mode: 0, + next_page_id: new_next, + commit_retain_policy_tag: policy_tag, + commit_retain_policy_value: policy_value, + }; + + let hk_clone = { self.db.hk.read().clone() }; + let new_slot = commit_header( + &*self.db.vfs, + &self.db.main_db_path, + &hk_clone, + &fields, + self.guard.active_slot, + self.db.page_size, + ) + .await?; + self.db.pager.commit_anchor(counter_anchor)?; + + // Apply pending segment side effects after the header is durable. + if !self.pending_segments.is_empty() { + self.db.vfs.mkdir_all("seg").await?; + self.db.vfs.sync_dir("seg").await?; + self.db.vfs.mkdir_all("seg/.tombstone").await?; + self.db.vfs.sync_dir("seg/.tombstone").await?; + for effect in &self.pending_segments { + match effect { + SegmentSideEffect::Promote { segment_id } => { + let staging = crate::segment::writer::staging_path(segment_id); + let live = crate::segment::writer::live_path(segment_id); + self.db.vfs.rename(&staging, &live).await?; + } + SegmentSideEffect::Tombstone { segment_id } => { + if self.db.segment_id_is_reader_pinned(*segment_id).await? { + self.db.pending_tombstones.lock().push(PendingTombstone { + segment_id: *segment_id, + commit_id: new_commit_id, + }); + } else { + let live = crate::segment::writer::live_path(segment_id); + let tomb = format!( + "seg/.tombstone/{}.{}", + crate::hex::to_hex_lower(segment_id), + new_commit_id, + ); + self.db.vfs.rename(&live, &tomb).await?; + } + } + } + } + self.db.vfs.sync_dir("seg").await?; + self.db.vfs.sync_dir("seg/.tombstone").await?; + } + + self.guard.root_page_id = new_root; + self.guard.next_page_id = new_next; + self.guard.active_slot = new_slot; + self.guard.seq = new_seq; + self.guard.latest_commit_id = new_commit_id; + self.guard.catalog_root_page_id = new_catalog_root; + self.guard.catalog_root_txn_id = new_commit_id; + self.guard.free_list_root_page_id = new_free_list_root; + // commit_history_root_page_id and commit_history_root_version are + // already updated inside write_commit_history_entry. + self.db.latest_commit.store(new_commit_id, Ordering::SeqCst); + self.committed_or_aborted = true; + + // The allocator cache is rebuilt from the durable free-list at the next + // `begin_write`, so nothing is handed off here. + + // Remove the spill tmp file (best-effort; error is non-fatal). + self.cleanup_spill_async().await; + + self.db + .spill_bytes_in_use + .store(0, std::sync::atomic::Ordering::Relaxed); + tracing::debug!( + name = "txn.commit", + commit_id = new_commit_id, + "write transaction committed" + ); + Ok(CommitId(new_commit_id)) + } +} + +fn page_size_log2(page_size: usize) -> Result { + match page_size { + 4096 => Ok(12), + 8192 => Ok(13), + 16384 => Ok(14), + 32768 => Ok(15), + 65536 => Ok(16), + _ => Err(PagedbError::Unsupported), + } +} + +/// Encode a `RetainPolicy` into the two header fields `(tag, value)`. +/// tag 0 = Count, tag 1 = Age (seconds), tag 2 = Unbounded, tag 3 = Disabled. +fn encode_retain_policy(policy: &crate::options::RetainPolicy) -> (u8, u64) { + match policy { + crate::options::RetainPolicy::Count(n) => (0, u64::from(*n)), + crate::options::RetainPolicy::Age(d) => (1, d.as_secs()), + crate::options::RetainPolicy::Unbounded => (2, 0), + crate::options::RetainPolicy::Disabled => (3, 0), + } +} diff --git a/src/txn/write/counter.rs b/src/txn/write/counter.rs new file mode 100644 index 0000000..64b92ed --- /dev/null +++ b/src/txn/write/counter.rs @@ -0,0 +1,75 @@ +//! `CounterRef` — durable monotonic named counters scoped to a `WriteTxn`. + +use crate::Result; +use crate::btree::BTree; +use crate::catalog::codec::Catalog; +use crate::errors::PagedbError; +use crate::vfs::Vfs; + +/// A handle to a named durable monotonic counter, scoped to a `WriteTxn`. +/// +/// The counter starts at `0` if it has never been written. Values are stored +/// as 8-byte little-endian `u64` catalog rows with key prefix `0x02`. +/// +/// Monotonicity guarantee: `set(v)` returns `PagedbError::Aborted` when +/// `v < current`; `increment_by` can never produce a value smaller than the +/// current one. Values only survive once the enclosing `WriteTxn::commit` +/// succeeds. +pub struct CounterRef<'a, V: Vfs + Clone> { + pub(super) catalog_tree: &'a mut BTree, + pub(super) main_tree: &'a mut BTree, + pub(super) key: Vec, +} + +impl CounterRef<'_, V> { + /// Return the current counter value, or `0` if no row exists yet. + pub async fn get(&self) -> Result { + match self.catalog_tree.get(&self.key).await? { + Some(v) => Catalog::decode_counter(&v), + None => Ok(0), + } + } + + /// Set the counter to `value`. Returns `PagedbError::Aborted` when + /// `value < current` (strict monotonicity). + pub async fn set(&mut self, value: u64) -> Result<()> { + let current = self.get().await?; + if value < current { + return Err(PagedbError::Aborted); + } + Self::sync_to(self.main_tree, self.catalog_tree); + let bytes = Catalog::encode_counter(value); + self.catalog_tree.put(&self.key, &bytes).await?; + Self::sync_from(self.catalog_tree, self.main_tree); + Ok(()) + } + + /// Increment the counter by `delta` and return the new value. Returns + /// `PagedbError::NonceCounterExhausted` on `u64` overflow. + pub async fn increment_by(&mut self, delta: u64) -> Result { + let current = self.get().await?; + let next = current + .checked_add(delta) + .ok_or(PagedbError::NonceCounterExhausted)?; + Self::sync_to(self.main_tree, self.catalog_tree); + let bytes = Catalog::encode_counter(next); + self.catalog_tree.put(&self.key, &bytes).await?; + Self::sync_from(self.catalog_tree, self.main_tree); + Ok(next) + } + + /// Advance the catalog allocator cursor to be at least as high as the + /// main tree's before a catalog write. + fn sync_to(main: &BTree, catalog: &mut BTree) { + let shared = main.next_page_id().max(catalog.next_page_id()); + catalog.set_next_page_id(shared); + } + + /// After a catalog write, propagate any advances back to the main tree. + fn sync_from(catalog: &BTree, main: &mut BTree) { + let c = catalog.next_page_id(); + if main.next_page_id() < c { + main.set_next_page_id(c); + } + } +} diff --git a/src/txn/write/mod.rs b/src/txn/write/mod.rs new file mode 100644 index 0000000..c6c630f --- /dev/null +++ b/src/txn/write/mod.rs @@ -0,0 +1,11 @@ +//! `WriteTxn` and its supporting types: the exclusive write session, spill +//! scratch storage, durable monotonic counters, and the commit path. + +mod commit; +mod counter; +mod spill; +mod txn; + +pub use counter::CounterRef; +pub use spill::{ScratchOffset, SpillScope}; +pub use txn::WriteTxn; diff --git a/src/txn/write/spill.rs b/src/txn/write/spill.rs new file mode 100644 index 0000000..20a9ec4 --- /dev/null +++ b/src/txn/write/spill.rs @@ -0,0 +1,249 @@ +//! Per-`WriteTxn` AEAD-encrypted spill scratch storage. + +use crate::Result; +use crate::crypto::aad::{Aad, AadFields}; +use crate::crypto::cipher::Cipher; +use crate::crypto::kdf::derive_spill_key; +use crate::crypto::nonce::Nonce; +use crate::errors::{PagedbError, QuotaKind}; +use crate::vfs::types::OpenMode; +use crate::vfs::{Vfs, VfsFile}; + +use super::txn::WriteTxn; + +/// Opaque handle returned by [`SpillScope::append`]. Pass to +/// [`SpillScope::read`] to decrypt and retrieve the bytes. +#[derive(Debug, Clone, Copy)] +pub struct ScratchOffset(u64); + +/// Metadata for one ciphertext chunk in the per-txn spill scratch file. +/// Stored in memory only; the tmp file is discarded at commit/abort. +#[derive(Clone)] +pub(crate) struct SpillSegmentMeta { + /// Byte offset of this chunk (ciphertext body start) in the tmp file. + pub offset: u64, + /// Length of the original plaintext in bytes. + pub plaintext_len: u32, + /// Length of ciphertext body (without the 16-byte tag) in bytes. + /// Total on-disk size for this chunk = `ciphertext_len + 16`. + pub ciphertext_len: u32, + /// The 12-byte nonce used to encrypt this chunk, stored verbatim so we + /// can reconstruct a `Nonce` on read without an additional lookup. + pub nonce_bytes: [u8; 12], +} + +/// A borrowed view into an active `WriteTxn` that offers AEAD-encrypted spill +/// storage in a per-transaction tmp file. +/// +/// The tmp file lives at `tmp/scratch-`. It is created lazily on the +/// first `append` call and removed at `commit` or `abort` (best-effort). +/// +/// Nonce scheme: `txn_seq_le6 ‖ segment_index_le6`. +/// - First 6 bytes: the low 6 bytes of `txn_seq` in little-endian order. +/// - Last 6 bytes: the low 6 bytes of the per-append segment index in +/// little-endian order. +/// +/// This guarantees uniqueness across all appends within a txn and across +/// independent txns (different `txn_seq`), without requiring a durable +/// nonce anchor (the tmp file is discarded before any subsequent txn can +/// reuse the same `txn_seq`). +pub struct SpillScope<'scope, 'db, V: Vfs + Clone> { + pub(super) txn: &'scope mut WriteTxn<'db, V>, +} + +impl SpillScope<'_, '_, V> { + /// Encrypt `bytes` under the per-txn spill key and append the ciphertext + /// plus 16-byte AEAD tag to the tmp file. Returns a `ScratchOffset` handle + /// usable with [`read`](Self::read). + /// + /// Returns `PagedbError::Quota { kind: QuotaKind::ScratchPages, … }` when + /// the cumulative ciphertext bytes (body + tag) would exceed the + /// `OpenOptions::scratch_bytes` budget. + pub async fn append(&mut self, bytes: &[u8]) -> Result { + let limit = self.txn.db.options.scratch_bytes as u64; + let segment_index = self.txn.spill_segments.len() as u64; + + // Extract immutable db fields before taking a mutable borrow for the + // cipher. Rust cannot prove these are disjoint borrows through the + // &mut reference, so we copy them out first. + let mk_epoch = self + .txn + .db + .mk_epoch + .load(std::sync::atomic::Ordering::SeqCst); + let realm_id = self.txn.db.realm_id; + let file_id = self.txn.db.file_id; + + // Build the per-append nonce: txn_seq_le6 || segment_index_le6. + let nonce = self.txn.next_spill_nonce(segment_index); + + // Derive the spill cipher lazily (mutably borrows txn, then releases). + self.txn.ensure_spill_cipher()?; + let cipher_id_byte = self + .txn + .spill_cipher + .as_ref() + .expect("just derived") + .id() + .as_byte(); + + // Build AAD: binds cipher_id, a spill sentinel page_kind (0xFE), + // mk_epoch, this segment's index (as page_id), realm_id, and file_id. + let aad = Aad::from_fields(AadFields { + cipher_id: cipher_id_byte, + page_kind: 0xFE, + mk_epoch, + page_id: segment_index, + realm_id, + segment_id: file_id, + }); + + let mut body = bytes.to_vec(); + let tag = self + .txn + .spill_cipher + .as_ref() + .expect("derived above") + .encrypt(&nonce, &aad, &mut body)?; + + // ciphertext body + 16-byte tag. + let pers_len = body.len() as u64 + 16; + let new_total = self.txn.spill_bytes_used.saturating_add(pers_len); + if new_total > limit { + return Err(PagedbError::quota( + self.txn.db.realm_id, + QuotaKind::ScratchPages, + new_total, + limit, + )); + } + + // Lazy: create tmp dir and file on first append. + let path = self.txn.ensure_spill_path().await?; + + let mut file = self.txn.db.vfs.open(&path, OpenMode::CreateOrOpen).await?; + let body_offset = self.txn.spill_bytes_used; + let tag_offset = body_offset + body.len() as u64; + file.write_at(body_offset, &body).await?; + file.write_at(tag_offset, &tag).await?; + file.sync().await?; + + let plaintext_len = u32::try_from(bytes.len()).map_err(|_| PagedbError::PayloadTooLarge)?; + let ciphertext_len = u32::try_from(body.len()).map_err(|_| PagedbError::PayloadTooLarge)?; + + self.txn.spill_segments.push(SpillSegmentMeta { + offset: body_offset, + plaintext_len, + ciphertext_len, + nonce_bytes: *nonce.as_bytes(), + }); + self.txn.spill_bytes_used = new_total; + self.txn + .db + .spill_bytes_in_use + .store(new_total, std::sync::atomic::Ordering::Relaxed); + + Ok(ScratchOffset(segment_index)) + } + + /// Decrypt and return the bytes previously written via + /// [`append`](Self::append) at `handle`. + pub async fn read(&self, handle: ScratchOffset) -> Result> { + let idx = usize::try_from(handle.0).map_err(|_| PagedbError::NotFound)?; + let meta = self + .txn + .spill_segments + .get(idx) + .ok_or(PagedbError::NotFound)? + .clone(); + let path = self.txn.spill_path.as_ref().ok_or(PagedbError::NotFound)?; + + let file = self.txn.db.vfs.open(path, OpenMode::Read).await?; + + let body_len = meta.ciphertext_len as usize; + let mut body = vec![0u8; body_len]; + let mut tag = [0u8; 16]; + file.read_at(meta.offset, &mut body).await?; + file.read_at(meta.offset + body_len as u64, &mut tag) + .await?; + + let cipher = self.txn.spill_cipher_readonly()?; + let nonce = Nonce::from_bytes(meta.nonce_bytes); + let aad = Aad::from_fields(AadFields { + cipher_id: cipher.id().as_byte(), + page_kind: 0xFE, + mk_epoch: self + .txn + .db + .mk_epoch + .load(std::sync::atomic::Ordering::SeqCst), + page_id: handle.0, + realm_id: self.txn.db.realm_id, + segment_id: self.txn.db.file_id, + }); + + cipher.decrypt(&nonce, &aad, &mut body, &tag)?; + body.truncate(meta.plaintext_len as usize); + Ok(body) + } +} + +impl<'db, V: Vfs + Clone> WriteTxn<'db, V> { + /// Return a [`SpillScope`] that writes AEAD-encrypted bytes to a per-txn + /// tmp file within the `scratch_bytes` budget from `OpenOptions`. + pub fn spill_scope(&mut self) -> SpillScope<'_, 'db, V> { + SpillScope { txn: self } + } + + /// Lazily derive and cache the per-txn spill cipher (AES-256-GCM keyed + /// with a spill-specific HKDF derivation from the master key). + pub(crate) fn ensure_spill_cipher(&mut self) -> Result<&Cipher> { + if self.spill_cipher.is_none() { + let pager_mk = self.db.pager.mk(); + let key = derive_spill_key(&pager_mk, &self.db.file_id, self.txn_seq)?; + self.spill_cipher = Some(Cipher::new_aes_gcm(&key)); + } + Ok(self.spill_cipher.as_ref().expect("just set")) + } + + /// Return a reference to the cached spill cipher for read paths (does NOT + /// lazily create — callers must have already called `ensure_spill_cipher` + /// or `append` at least once, which guarantees the cipher exists). + pub(crate) fn spill_cipher_readonly(&self) -> Result<&Cipher> { + self.spill_cipher.as_ref().ok_or(PagedbError::NotFound) + } + + /// Lazily create the `tmp/` directory and return the spill file path. + pub(crate) async fn ensure_spill_path(&mut self) -> Result { + if self.spill_path.is_none() { + self.db.vfs.mkdir_all("tmp").await?; + let path = format!("tmp/scratch-{}", self.txn_seq); + self.spill_path = Some(path); + } + Ok(self.spill_path.clone().expect("just set")) + } + + /// Build the per-append nonce deterministically from `(txn_seq, segment_index)`. + /// + /// Layout: `txn_seq_le6 ‖ segment_index_le6` (6 + 6 = 12 bytes). + /// Uniqueness: nonces are unique per-txn (distinct `segment_index`) and + /// across txns (distinct `txn_seq` in first 6 bytes). No durable anchor + /// is needed because the tmp file is discarded before any key reuse + /// could matter. + pub(crate) fn next_spill_nonce(&self, segment_index: u64) -> Nonce { + let mut bytes = [0u8; 12]; + let seq_le = self.txn_seq.to_le_bytes(); + let idx_le = segment_index.to_le_bytes(); + bytes[..6].copy_from_slice(&seq_le[..6]); + bytes[6..].copy_from_slice(&idx_le[..6]); + Nonce::from_bytes(bytes) + } + + /// Remove the spill tmp file if it was created. Errors are swallowed + /// (best-effort cleanup); the file is transient anyway. + pub(crate) async fn cleanup_spill_async(&mut self) { + if let Some(path) = self.spill_path.take() { + let _ = self.db.vfs.remove(&path).await; + } + } +} diff --git a/src/txn/write/txn.rs b/src/txn/write/txn.rs new file mode 100644 index 0000000..bafaff2 --- /dev/null +++ b/src/txn/write/txn.rs @@ -0,0 +1,356 @@ +//! `WriteTxn` — exclusive write session backed by a `CoW` B+ tree. On commit, +//! flushes dirty pages, writes the new A/B header, and advances the +//! visibility commit id. The commit body lives in [`super::commit`]. + +use std::sync::atomic::Ordering; + +use tokio::sync::MutexGuard; + +use crate::btree::BTree; +use crate::catalog::codec::{Catalog, RealmQuotas, SegmentMeta}; +use crate::crypto::cipher::Cipher; +use crate::errors::{PagedbError, QuotaKind}; +use crate::vfs::Vfs; +use crate::{RealmId, Result}; + +use super::super::db::{Db, WriterState}; +use super::counter::CounterRef; +use super::spill::SpillSegmentMeta; + +/// Deferred filesystem operation applied after the A/B header is durable. +pub(crate) enum SegmentSideEffect { + Promote { segment_id: [u8; 16] }, + Tombstone { segment_id: [u8; 16] }, +} + +/// An exclusive write transaction. At most one `WriteTxn` exists per `Db` at +/// any time — the writer mutex enforces this. Either `commit` or `abort` must +/// be called; if the `WriteTxn` is dropped without either, dirty pages are +/// silently discarded (equivalent to abort). +pub struct WriteTxn<'db, V: Vfs + Clone> { + pub(super) db: &'db Db, + pub(super) guard: MutexGuard<'db, WriterState>, + pub(super) btree: BTree, + pub(super) catalog_tree: BTree, + pub(super) pending_segments: Vec, + pub(super) committed_or_aborted: bool, + /// Monotonic per-txn sequence number; assigned at `begin` from + /// `Db::txn_seq.fetch_add(1, Relaxed) + 1`. The first `WriteTxn` on a + /// fresh Db gets `txn_seq == 1`, making `tmp/scratch-1` predictable in + /// tests. + pub(crate) txn_seq: u64, + /// Lazily derived spill cipher. `None` until the first `spill_scope` append. + pub(crate) spill_cipher: Option, + /// Lazily created path to the per-txn spill tmp file (`tmp/scratch-`). + pub(crate) spill_path: Option, + /// Cumulative bytes (ciphertext body + tag) written to the spill file. + pub(crate) spill_bytes_used: u64, + /// Per-append metadata used by `SpillScope::read` to reconstruct AAD/nonce. + pub(crate) spill_segments: Vec, + /// The durable free-list's `(commit_id, page_id)` entries as loaded at + /// begin. The commit path rewrites the chain from these (minus the pages + /// reused this txn, plus the pages freed this txn). + pub(crate) free_set_loaded: Vec<(u64, u64)>, + /// Page ids the old free-list chain occupied at begin. They become free + /// once this commit's new chain supersedes them. + pub(crate) old_chain_pages: Vec, + /// Reclamation floor at begin: free-list entries tagged below this are + /// drainable (no snapshot pins them). The reader-stall policy is evaluated + /// at commit against only the entries at/above it — the backlog genuinely + /// stuck behind a reader pin — not the drainable remainder. + pub(crate) reclaim_floor: u64, +} + +impl<'db, V: Vfs + Clone> WriteTxn<'db, V> { + pub(crate) async fn begin(db: &'db Db) -> Result> { + let guard = db.writer.lock().await; + + // Pages freed *within this txn* must not be recycled within it while a + // reader is pinned (they may still be live in a prior snapshot). The + // reuse threshold gates the per-txn freed list for that; the shared + // cache is gated separately (it holds only floor-safe pages — below). + let min_reader = { + let readers = db.tracked_readers.lock(); + readers.iter().map(|r| r.commit_id.0).min() + }; + let reuse_threshold = if min_reader.is_none() { + 0 + } else { + guard.next_page_id + }; + + // Load the durable free-list and rebuild the shared allocator cache with + // exactly the pages below the reclamation floor — the older of the + // oldest live-reader pin and the oldest retained commit-history root. + // Those are observable by no snapshot, so they are safe to recycle now. + let history_floor = db + .oldest_retained_history_commit(guard.commit_history_root_page_id, guard.next_page_id) + .await?; + let floor = min_reader + .unwrap_or(u64::MAX) + .min(history_floor.map_or(u64::MAX, |h| h.saturating_add(1))); + let (free_set_loaded, old_chain_pages) = crate::pager::freelist::read_chain( + &db.pager, + db.realm_id, + guard.free_list_root_page_id, + ) + .await?; + { + let mut cache = db.free_page_cache.lock(); + cache.clear(); + for (cid, pid) in &free_set_loaded { + if *cid < floor { + cache.push(*pid); + } + } + } + db.free_page_consumed.lock().clear(); + + let mut btree = BTree::open( + db.pager.clone(), + db.realm_id, + guard.root_page_id, + guard.next_page_id, + db.page_size, + ); + btree.set_reuse_threshold(reuse_threshold); + btree.set_free_page_cache(db.free_page_cache.clone()); + btree.set_free_page_consumed(db.free_page_consumed.clone()); + let mut catalog_tree = BTree::open( + db.pager.clone(), + db.realm_id, + guard.catalog_root_page_id, + guard.next_page_id, + db.page_size, + ); + catalog_tree.set_reuse_threshold(reuse_threshold); + catalog_tree.set_free_page_cache(db.free_page_cache.clone()); + catalog_tree.set_free_page_consumed(db.free_page_consumed.clone()); + // Assign a txn_seq starting from 1: fetch_add returns the old value (0 + // for the first call), so we add 1 to produce 1-based ids. + let txn_seq = db.txn_seq.fetch_add(1, Ordering::Relaxed) + 1; + Ok(Self { + db, + guard, + btree, + catalog_tree, + pending_segments: Vec::new(), + committed_or_aborted: false, + txn_seq, + spill_cipher: None, + spill_path: None, + spill_bytes_used: 0, + spill_segments: Vec::new(), + free_set_loaded, + old_chain_pages, + reclaim_floor: floor, + }) + } + + pub async fn get(&self, key: &[u8]) -> Result>> { + self.btree.get(key).await + } + + pub async fn put(&mut self, key: &[u8], value: &[u8]) -> Result<()> { + self.btree.put(key, value).await + } + + /// Append a key-value pair under the monotonic-key invariant. + /// + /// Subsequent calls within the same `WriteTxn` skip the + /// `path_to_leaf_for_key` descent by reusing the cached rightmost + /// path; splits and explicitly-invalidating operations (regular `put`, + /// `delete`) force a re-descent on the next call. + /// + /// Intended for op-logs, time-series indexes, FTS posting-list builds + /// — any workload where the embedder can guarantee monotonically + /// increasing keys. + /// + /// # Errors + /// + /// Returns [`PagedbError::AppendNotMonotonic`] if `key` is not + /// strictly greater than the previously-appended key in this txn. + pub async fn put_append(&mut self, key: &[u8], value: &[u8]) -> Result<()> { + self.btree.put_append(key, value).await + } + + pub async fn delete(&mut self, key: &[u8]) -> Result { + self.btree.delete(key).await + } + + pub async fn put_batch(&mut self, sorted: Vec<(Vec, Vec)>) -> Result<()> { + self.btree.put_batch(sorted).await + } + + pub async fn delete_batch(&mut self, sorted: Vec>) -> Result<()> { + self.btree.delete_batch(sorted).await + } + + pub async fn delete_range(&mut self, start: &[u8], end: &[u8]) -> Result { + self.btree.delete_range(start, end).await + } + + /// Return a `CounterRef` scoped to this transaction for the named counter. + /// + /// The returned handle borrows `self` mutably for its lifetime. Use the + /// counter, drop the handle, then continue with other transaction + /// operations. The name must be at most `MAX_SEGMENT_NAME_LEN` bytes + /// (`PagedbError::NameTooLong` otherwise). + pub fn counter<'tx>(&'tx mut self, name: &str) -> Result> { + let key = Catalog::counter_key(name.as_bytes())?; + Ok(CounterRef { + catalog_tree: &mut self.catalog_tree, + main_tree: &mut self.btree, + key, + }) + } + + /// Register a segment under `name` in the catalog and schedule promotion + /// of its staging file to the live path on commit. + pub async fn link_segment(&mut self, name: &str, meta: &SegmentMeta) -> Result<()> { + let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; + if self.catalog_tree.get(&key).await?.is_some() { + return Err(PagedbError::AlreadyLinked); + } + self.enforce_segment_bytes_quota(meta.realm_id, meta.total_bytes, 0) + .await?; + let value = Catalog::encode_segment_meta(meta); + self.sync_allocator_to_catalog(); + self.catalog_tree.put(&key, &value).await?; + self.sync_allocator_from_catalog(); + self.pending_segments.push(SegmentSideEffect::Promote { + segment_id: meta.segment_id, + }); + Ok(()) + } + + /// Remove the catalog row for `name` and schedule a tombstone rename on commit. + pub async fn unlink_segment(&mut self, name: &str) -> Result<()> { + let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; + let value = self + .catalog_tree + .get(&key) + .await? + .ok_or(PagedbError::NotLinked)?; + let meta = Catalog::decode_segment_meta(&value)?; + self.sync_allocator_to_catalog(); + let removed = self.catalog_tree.delete(&key).await?; + self.sync_allocator_from_catalog(); + if !removed { + return Err(PagedbError::NotLinked); + } + self.pending_segments.push(SegmentSideEffect::Tombstone { + segment_id: meta.segment_id, + }); + Ok(()) + } + + /// Atomically swap the segment recorded under `name`: tombstone the old + /// segment id and promote `new_meta`'s staging file on commit. + pub async fn replace_segment(&mut self, name: &str, new_meta: &SegmentMeta) -> Result<()> { + let key = Catalog::segment_key(self.db.realm_id, name.as_bytes())?; + let existing = self + .catalog_tree + .get(&key) + .await? + .ok_or(PagedbError::NotLinked)?; + let old_meta = Catalog::decode_segment_meta(&existing)?; + self.enforce_segment_bytes_quota( + new_meta.realm_id, + new_meta.total_bytes, + old_meta.total_bytes, + ) + .await?; + let value = Catalog::encode_segment_meta(new_meta); + self.sync_allocator_to_catalog(); + self.catalog_tree.put(&key, &value).await?; + self.sync_allocator_from_catalog(); + self.pending_segments.push(SegmentSideEffect::Tombstone { + segment_id: old_meta.segment_id, + }); + self.pending_segments.push(SegmentSideEffect::Promote { + segment_id: new_meta.segment_id, + }); + Ok(()) + } + + /// Check if the realm's committed segment bytes plus `new_bytes` minus + /// `delta_remove_bytes` would exceed the configured cap. Returns `Ok(())` + /// when no cap is set or the projected total is within the limit. + async fn enforce_segment_bytes_quota( + &self, + realm: RealmId, + new_bytes: u64, + delta_remove_bytes: u64, + ) -> Result<()> { + let quota_key = Catalog::quota_key(realm); + let quotas = match self.catalog_tree.get("a_key).await? { + Some(v) => Catalog::decode_realm_quotas(&v)?, + None => RealmQuotas::default(), + }; + let Some(limit) = quotas.max_segment_bytes else { + return Ok(()); + }; + // Scan all catalog segment rows for this realm to sum committed bytes. + let mut prefix = Vec::with_capacity(17); + prefix.push(0x01u8); // CatalogRowKind::Segment + prefix.extend_from_slice(&realm.0); + let mut end = prefix.clone(); + end.push(0xFF); + let rows = self.catalog_tree.collect_range(&prefix, &end).await?; + let mut committed: u64 = 0; + for (_, v) in rows { + let meta = Catalog::decode_segment_meta(&v)?; + committed = committed.saturating_add(meta.total_bytes); + } + let after_remove = committed.saturating_sub(delta_remove_bytes); + let projected = after_remove.saturating_add(new_bytes); + if projected > limit { + return Err(PagedbError::quota( + realm, + QuotaKind::SegmentBytes, + projected, + limit, + )); + } + Ok(()) + } + + /// Discard all in-flight dirty pages without writing anything durable. + /// The spill tmp file (if created) is removed before returning (best-effort; + /// errors are ignored). + pub async fn abort(mut self) { + tracing::debug!(name = "txn.abort", "write transaction aborted"); + self.db.pager.discard_dirty_main(self.db.realm_id); + self.committed_or_aborted = true; + self.db + .spill_bytes_in_use + .store(0, std::sync::atomic::Ordering::Relaxed); + self.cleanup_spill_async().await; + } + + /// Before a catalog operation: ensure the catalog tree's allocator cursor + /// is at least as high as the main tree's, so neither tree allocates the + /// same page id. + pub(super) fn sync_allocator_to_catalog(&mut self) { + let main_next = self.btree.next_page_id(); + let cat_next = self.catalog_tree.next_page_id(); + let shared = main_next.max(cat_next); + self.catalog_tree.set_next_page_id(shared); + } + + /// After a catalog operation: propagate any catalog allocation advances + /// back to the main tree so subsequent main-tree allocations stay above. + pub(super) fn sync_allocator_from_catalog(&mut self) { + let cat_next = self.catalog_tree.next_page_id(); + self.btree.set_next_page_id(cat_next); + } +} + +impl Drop for WriteTxn<'_, V> { + fn drop(&mut self) { + if !self.committed_or_aborted { + self.db.pager.discard_dirty_main(self.db.realm_id); + } + } +} diff --git a/src/vfs/gcd/file.rs b/src/vfs/gcd/file.rs index c6fdae1..78ca888 100644 --- a/src/vfs/gcd/file.rs +++ b/src/vfs/gcd/file.rs @@ -92,6 +92,117 @@ impl Drop for GcdFile { } } +/// Submit a `dispatch_io` read and return immediately. Kept synchronous on +/// purpose: the handler `RcBlock` and the raw block pointer derived from it are +/// `!Send`, so they must never be live across an await point in the calling +/// future (`VfsFile::read_at` must return a `Send` future). The handler +/// accumulates chunks and, on completion, sends the result through `tx`. +fn submit_read( + channel: &DispatchIO, + queue: &DispatchQueue, + offset: u64, + len: usize, + tx: tokio::sync::oneshot::Sender>>, +) { + let accum: Arc>> = Arc::new(Mutex::new(Vec::with_capacity(len))); + let sender = Arc::new(Mutex::new(Some(tx))); + + // block2 cannot encode `bool` or `*mut DispatchData` directly (the + // dispatch_object! types don't implement `RefEncode`). We declare the + // closure with ABI-compatible stand-ins — `u8` for bool and `*mut c_void` + // for DispatchData — and transmute the resulting block pointer to the + // dispatch_io_handler_t signature. dispatch2's own `DispatchData::to_vec` + // uses the same workaround (see data.rs:114). + let handler: RcBlock = + RcBlock::new(move |done: u8, data: *mut c_void, error: libc::c_int| { + if !data.is_null() { + let d: &DispatchData = + // SAFETY: dispatch guarantees `data` (when non-null) is a + // valid `DispatchData` retained for the handler's duration. + unsafe { &*data.cast::() }; + let bytes = d.to_vec(); + accum.lock().extend_from_slice(&bytes); + } + if done != 0 { + let result = if error != 0 { + Err(std::io::Error::from_raw_os_error(error)) + } else { + Ok(std::mem::take(&mut *accum.lock())) + }; + if let Some(s) = sender.lock().take() { + let _ = s.send(result); + } + } + }); + + // SAFETY: transmute is from the stand-in block signature to the typedef + // declared by dispatch2 — the ABI is identical because `bool` and `u8` + // share the same one-byte ABI, and a `*mut DispatchData` is bit-identical + // to a `*mut c_void`. + let handler_ptr: *mut DynBlock = unsafe { + std::mem::transmute::< + *mut Block, + *mut DynBlock, + >(RcBlock::as_ptr(&handler)) + }; + + // SAFETY: channel and queue are valid (owned by the caller's `GcdFile`); + // the handler block is retained by libdispatch on submission. + unsafe { + #[allow(clippy::cast_possible_wrap)] + channel.read(offset as libc::off_t, len, queue, handler_ptr); + } + // libdispatch has retained the block internally; we can drop our Rc. + drop(handler); +} + +/// Submit a `dispatch_io` write and return immediately. Synchronous for the +/// same reason as [`submit_read`]: the handler block, its raw pointer, and the +/// `DispatchData` are all `!Send` and must not cross an await point. +fn submit_write( + channel: &DispatchIO, + queue: &DispatchQueue, + offset: u64, + buf: &[u8], + tx: tokio::sync::oneshot::Sender>, +) { + let data = DispatchData::from_bytes(buf); + let sender = Arc::new(Mutex::new(Some(tx))); + + // Same bool→u8 / *mut DispatchData→*mut c_void workaround as `submit_read`. + let handler: RcBlock = RcBlock::new( + move |done: u8, _remaining: *mut c_void, error: libc::c_int| { + if done != 0 { + let result = if error != 0 { + Err(std::io::Error::from_raw_os_error(error)) + } else { + Ok(()) + }; + if let Some(s) = sender.lock().take() { + let _ = s.send(result); + } + } + }, + ); + + // SAFETY: ABI-compatible transmute; see `submit_read`. + let handler_ptr: *mut DynBlock = unsafe { + std::mem::transmute::< + *mut Block, + *mut DynBlock, + >(RcBlock::as_ptr(&handler)) + }; + + // SAFETY: channel/queue/data all valid; libdispatch retains both the data + // object and the handler block for the operation's duration. + unsafe { + #[allow(clippy::cast_possible_wrap)] + channel.write(offset as libc::off_t, &data, queue, handler_ptr); + } + drop(handler); + drop(data); +} + impl VfsFile for GcdFile { async fn read_at(&self, offset: u64, buf: &mut [u8]) -> Result { if buf.is_empty() { @@ -99,60 +210,12 @@ impl VfsFile for GcdFile { } let len = buf.len(); let (tx, rx) = tokio::sync::oneshot::channel::>>(); - let accum: Arc>> = Arc::new(Mutex::new(Vec::with_capacity(len))); - let sender = Arc::new(Mutex::new(Some(tx))); - - // block2 cannot encode `bool` or `*mut DispatchData` directly (the - // dispatch_object! types don't implement `RefEncode`). We declare the - // closure with ABI-compatible stand-ins — `u8` for bool and - // `*mut c_void` for DispatchData — and transmute the resulting block - // pointer to the dispatch_io_handler_t signature. dispatch2's own - // `DispatchData::to_vec` uses the same workaround (see data.rs:114). - let accum_for_block = Arc::clone(&accum); - let sender_for_block = Arc::clone(&sender); - let handler: RcBlock = - RcBlock::new(move |done: u8, data: *mut c_void, error: libc::c_int| { - if !data.is_null() { - let d: &DispatchData = - // SAFETY: dispatch guarantees `data` (when non-null) - // is a valid `DispatchData` retained for the - // handler's duration. - unsafe { &*data.cast::() }; - let bytes = d.to_vec(); - accum_for_block.lock().extend_from_slice(&bytes); - } - if done != 0 { - let result = if error != 0 { - Err(std::io::Error::from_raw_os_error(error)) - } else { - Ok(std::mem::take(&mut *accum_for_block.lock())) - }; - if let Some(s) = sender_for_block.lock().take() { - let _ = s.send(result); - } - } - }); - - // SAFETY: transmute is from the stand-in block signature to the - // typedef declared by dispatch2 — the ABI is identical because `bool` - // and `u8` share the same one-byte ABI, and a `*mut DispatchData` is - // bit-identical to a `*mut c_void`. - let handler_ptr: *mut DynBlock = unsafe { - std::mem::transmute::< - *mut Block, - *mut DynBlock, - >(RcBlock::as_ptr(&handler)) - }; - - // SAFETY: the channel and queue are valid (held by `self`); the - // handler block is retained by libdispatch on submission. - unsafe { - #[allow(clippy::cast_possible_wrap)] - self.channel - .read(offset as libc::off_t, len, &self.queue, handler_ptr); - } - // libdispatch has retained the block internally; we can drop our Rc. - drop(handler); + // The handler block and its raw block pointer are `!Send`; submitting + // from a synchronous helper keeps them out of this future's state + // machine, so the future stays `Send` as `VfsFile` requires. Once + // submitted, libdispatch owns the operation and reports back through the + // oneshot, which is all we await here. + submit_read(&self.channel, &self.queue, offset, len, tx); let data = rx .await @@ -184,45 +247,9 @@ impl VfsFile for GcdFile { return Ok(0); } let len = buf.len(); - let data = DispatchData::from_bytes(buf); let (tx, rx) = tokio::sync::oneshot::channel::>(); - let sender = Arc::new(Mutex::new(Some(tx))); - - // Same bool→u8 / *mut DispatchData→*mut c_void workaround as - // `read_at`. See comments there. - let sender_for_block = Arc::clone(&sender); - let handler: RcBlock = RcBlock::new( - move |done: u8, _remaining: *mut c_void, error: libc::c_int| { - if done != 0 { - let result = if error != 0 { - Err(std::io::Error::from_raw_os_error(error)) - } else { - Ok(()) - }; - if let Some(s) = sender_for_block.lock().take() { - let _ = s.send(result); - } - } - }, - ); - - // SAFETY: ABI-compatible transmute; see `read_at`. - let handler_ptr: *mut DynBlock = unsafe { - std::mem::transmute::< - *mut Block, - *mut DynBlock, - >(RcBlock::as_ptr(&handler)) - }; - - // SAFETY: channel/queue/data all valid; libdispatch retains both the - // data object and the handler block for the operation's duration. - unsafe { - #[allow(clippy::cast_possible_wrap)] - self.channel - .write(offset as libc::off_t, &data, &self.queue, handler_ptr); - } - drop(handler); - drop(data); + // `!Send` handler/data confined to a synchronous helper; see `read_at`. + submit_write(&self.channel, &self.queue, offset, buf, tx); rx.await .map_err(|_| PagedbError::Io(std::io::Error::other("dispatch_io write cancelled")))? diff --git a/src/vfs/tokio_backend.rs b/src/vfs/tokio_backend.rs index 80cf16c..4bf8644 100644 --- a/src/vfs/tokio_backend.rs +++ b/src/vfs/tokio_backend.rs @@ -419,6 +419,18 @@ impl Vfs for TokioVfs { let dir = match std::fs::File::open(&p) { Ok(d) => d, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + // Opening a directory as a file handle is unsupported on some + // platforms — notably Windows, where it fails with PermissionDenied + // ("Access is denied") at open, before sync_all is ever reached. + // Directory fsync is best-effort there, so treat it as a no-op. + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::Unsupported | std::io::ErrorKind::PermissionDenied + ) => + { + return Ok(()); + } Err(e) => return Err(PagedbError::Io(e)), }; match dir.sync_all() { diff --git a/tests/compaction_incremental.rs b/tests/compaction_incremental.rs index 8786a69..83d1d62 100644 --- a/tests/compaction_incremental.rs +++ b/tests/compaction_incremental.rs @@ -222,3 +222,111 @@ async fn compact_now_completes_fully() { ); } } + +// ─── Test: history is consistently discarded across compact_step (no dangling +// commit-history root in the durable header) ──────────────────────────── +#[tokio::test(flavor = "current_thread")] +async fn compact_step_reopen_history_consistent() { + let vfs = MemVfs::new(); + { + // Default options retain commit history (Count(1024)). Build several + // commits so a history tree exists, then incrementally compact through + // intermediate steps and a final dense repack. + let db = Db::open_internal(vfs.clone(), KEK, PAGE, REALM) + .await + .unwrap(); + for round in 0u32..15 { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..30 { + w.put(format!("k{round:02}_{i:04}").as_bytes(), &[round as u8; 64]) + .await + .unwrap(); + } + w.commit().await.unwrap(); + } + let budget = CompactBudget::new(5, 10_000); // small → intermediate steps + final + let mut iters = 0; + loop { + let p = db.compact_step(budget).await.unwrap(); + iters += 1; + assert!(iters < 10_000, "compaction did not converge"); + if !p.more_work { + break; + } + } + // Db drops here. + } + + // Reopen. Before the fix, the header still pointed at the pre-repack + // commit-history root (overwritten/truncated by the dense repack), so the + // next write — which opens the history tree at that root — corrupted or + // errored. It must now be a clean reset (root = 0). + let db = Db::open_existing(vfs.clone(), KEK, PAGE, REALM) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + w.put(b"after-reopen", b"v").await.unwrap(); + w.commit().await.unwrap(); + } + let r = db.begin_read().await.unwrap(); + assert_eq!( + r.get(b"after-reopen").await.unwrap().as_deref(), + Some(&b"v"[..]) + ); + // Pre-compaction data survived the repack. + assert!( + r.get(b"k00_0000").await.unwrap().is_some(), + "data written before compaction must survive" + ); +} + +// ─── Test: an intermediate compact_step preserves the durable free-list ────── +#[tokio::test(flavor = "current_thread")] +async fn compact_step_preserves_free_list() { + // No commit history, so freed pages are immediately reclaimable and tracked + // in the durable free-list with no reader/history pinning them. + let opts = pagedb::options::OpenOptions::default() + .with_commit_history_retain(pagedb::options::RetainPolicy::Disabled); + let db = Db::open_internal_with_options(MemVfs::new(), KEK, PAGE, REALM, opts) + .await + .unwrap(); + + // Build a working set, then delete most of it to populate the free-list. + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..300 { + w.put(format!("k{i:05}").as_bytes(), &[1u8; 128]) + .await + .unwrap(); + } + w.commit().await.unwrap(); + } + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..250 { + w.delete(format!("k{i:05}").as_bytes()).await.unwrap(); + } + w.commit().await.unwrap(); + } + let pending_before = db.stats().await.unwrap().free_list_pending_entries; + assert!( + pending_before > 0, + "setup should have populated the free-list; got {pending_before}" + ); + + // One intermediate step (small budget so it is NOT the final batch). + let prog = db + .compact_step(CompactBudget::new(5, 10_000)) + .await + .unwrap(); + assert!(prog.more_work, "small budget should leave more work"); + + // The pre-existing free-list must survive the intermediate step, not be + // wiped — those pages are still reusable by ordinary writes. + let pending_after = db.stats().await.unwrap().free_list_pending_entries; + assert!( + pending_after >= pending_before, + "intermediate compact_step wiped the durable free-list: {pending_before} -> {pending_after}" + ); +} diff --git a/tests/deferred_free_reclaim.rs b/tests/deferred_free_reclaim.rs new file mode 100644 index 0000000..5ab6247 --- /dev/null +++ b/tests/deferred_free_reclaim.rs @@ -0,0 +1,223 @@ +//! Deferred-free reclamation on the commit path. +//! +//! When no reader pins an old snapshot, pages freed by a commit are eligible +//! for immediate reuse. A store that rewrites a bounded working set over many +//! one-commit-per-write transactions must therefore stay bounded in both +//! file size and deferred-free backlog depth — the freed pages have to flow +//! back to the allocator instead of accumulating in the deferred-free queue. +//! +//! These tests pin the *bounded* contract: they rewrite a constant key set +//! across a growing number of commits and assert that neither the file's +//! high-water mark nor the deferred-free backlog grows in proportion to the +//! commit count. + +use pagedb::options::{OpenOptions, RetainPolicy}; +use pagedb::vfs::memory::MemVfs; +use pagedb::{Db, RealmId}; + +const KEK: [u8; 32] = [0x2Au8; 32]; +const REALM: RealmId = RealmId::new([0x5Cu8; 16]); +const PAGE: usize = 4096; + +/// A constant working set: the same keys rewritten every round so that the +/// live data size never changes — only the commit count does. +const LIVE_KEYS: u32 = 20; +const VALUE: [u8; 256] = [0xABu8; 256]; + +/// These tests isolate free-page *reuse*, so commit history is disabled: a +/// retained historical root legitimately pins the pages of every prior commit +/// (that's the point of retention), which would mask reclamation. Reclamation +/// under a *bounded* history window is covered separately by +/// [`bounded_history_reuses_pages_after_window`]. +fn no_history_opts() -> OpenOptions { + OpenOptions::default().with_commit_history_retain(RetainPolicy::Disabled) +} + +async fn fresh_db() -> Db { + Db::open_internal_with_options(MemVfs::new(), KEK, PAGE, REALM, no_history_opts()) + .await + .unwrap() +} + +/// Rewrite the whole working set in one transaction, one commit per round. +async fn overwrite_round(db: &Db, round: u32) { + let mut w = db.begin_write().await.unwrap(); + for i in 0..LIVE_KEYS { + let mut v = VALUE; + v[0] = (round % 251) as u8; + w.put(format!("k{i:04}").as_bytes(), &v).await.unwrap(); + } + w.commit().await.unwrap(); +} + +/// With no readers pinning, the deferred-free backlog must not accumulate in +/// proportion to the number of commits: eligible freed pages drain back to the +/// allocator on commit. A constant working set rewritten N times should leave +/// only a bounded backlog, not one that scales with N. +#[tokio::test(flavor = "current_thread")] +async fn sustained_overwrite_commits_drain_deferred_free() { + let db = fresh_db().await; + + // Seed the working set. + overwrite_round(&db, 0).await; + + // No reader is ever opened, so every freed page is immediately eligible + // for reuse. Run enough rounds that an un-drained queue would be obvious. + for round in 1..=200 { + overwrite_round(&db, round).await; + } + + let stats = db.stats().await.unwrap(); + assert!( + stats.free_list_pending_entries < 64, + "deferred-free backlog must stay bounded when no readers pin — it must \ + drain on commit, not accumulate one entry per freed page; after 200 \ + commits of a {LIVE_KEYS}-key working set the backlog was {} entries", + stats.free_list_pending_entries, + ); +} + +/// Rewriting a constant working set must reuse freed pages, so the file's +/// high-water mark (`next_page_id`) stays bounded regardless of how many +/// commits were issued. Today the allocator only bump-allocates across +/// commits, so `next_page_id` grows linearly with the commit count. +#[tokio::test(flavor = "current_thread")] +async fn sustained_overwrite_commits_keep_file_bounded() { + let db = fresh_db().await; + + overwrite_round(&db, 0).await; + + for round in 1..=200 { + overwrite_round(&db, round).await; + } + + let stats = db.stats().await.unwrap(); + // The working set is ~20 keys × 256 B; the entire live tree fits in a + // double-digit page count. The bound here is a constant — it must NOT + // scale with the 200 commits issued. + assert!( + stats.main_db_next_page_id < 200, + "file high-water mark must stay bounded for a constant working set; \ + after 200 one-commit-per-write rounds next_page_id was {} (linear \ + growth means freed pages were never reused)", + stats.main_db_next_page_id, + ); +} + +/// The bound must hold independent of commit count: doubling the number of +/// commits over the same working set must not roughly double the file size. +/// This is the direct guard against the super-linear growth reported. +#[tokio::test(flavor = "current_thread")] +async fn file_growth_is_independent_of_commit_count() { + let measure = |rounds: u32| async move { + let db = fresh_db().await; + overwrite_round(&db, 0).await; + for round in 1..=rounds { + overwrite_round(&db, round).await; + } + db.stats().await.unwrap().main_db_next_page_id + }; + + let at_100 = measure(100).await; + let at_400 = measure(400).await; + + // 4× the commits over the same data must not meaningfully grow the file. + assert!( + at_400 <= at_100 + 16, + "file size scales with commit count (super-linear growth): \ + next_page_id was {at_100} after 100 commits and {at_400} after 400 \ + commits of the same {LIVE_KEYS}-key working set", + ); +} + +/// With a *bounded* commit-history window, pages reachable only from commits +/// older than the window become reclaimable once those commits are pruned, so +/// sustained writes stay bounded a constant factor above the window size — +/// they do not grow without bound the way the unbounded deferred-free queue +/// did. This is the reported default-configuration daemon scenario. +#[tokio::test(flavor = "current_thread")] +async fn bounded_history_reuses_pages_after_window() { + const WINDOW: u32 = 8; + let opts = OpenOptions::default().with_commit_history_retain(RetainPolicy::Count(WINDOW)); + let db = Db::open_internal_with_options(MemVfs::new(), KEK, PAGE, REALM, opts) + .await + .unwrap(); + + overwrite_round(&db, 0).await; + for round in 1..=300 { + overwrite_round(&db, round).await; + } + + let stats = db.stats().await.unwrap(); + // Live working set + a bounded history window of CoW deltas + the bounded + // deferred-free queue that holds the window's not-yet-prunable frees. This + // is a small constant — emphatically not the ~900 pages 300 unreclaimed + // commits would produce. + assert!( + stats.main_db_next_page_id < 200, + "bounded history must keep the file bounded; after 300 commits with a \ + {WINDOW}-commit window next_page_id was {}", + stats.main_db_next_page_id, + ); +} + +/// The free list is durable: pages freed before an *unclean* shutdown (no +/// compaction, no graceful drain) are recovered on reopen and recycled, so the +/// file does not grow when the freed space is re-used after a restart. This is +/// the crash-durability guarantee — under an in-memory-only free pool those +/// pages would be orphaned until a compaction. +#[tokio::test(flavor = "current_thread")] +async fn free_list_survives_unclean_reopen() { + let vfs = MemVfs::new(); + + // Write a working set, then delete it — freeing many pages into the durable + // free list — and drop the handle without compacting or draining. + let high_water_before; + { + let db = Db::open_internal_with_options(vfs.clone(), KEK, PAGE, REALM, no_history_opts()) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..400 { + w.put(format!("k{i:05}").as_bytes(), &VALUE).await.unwrap(); + } + w.commit().await.unwrap(); + } + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..400 { + w.delete(format!("k{i:05}").as_bytes()).await.unwrap(); + } + w.commit().await.unwrap(); + } + high_water_before = db.stats().await.unwrap().main_db_next_page_id; + let pending = db.stats().await.unwrap().free_list_pending_entries; + assert!( + pending > 20, + "deletes should have freed pages into the durable free list; got {pending}" + ); + // Drop without compaction or graceful shutdown — simulates a crash. + } + + // Reopen and write a fresh working set. With the free list recovered from + // disk, these allocations recycle the freed pages instead of extending the + // file past where it already was. + let db = Db::open_existing_with_options(vfs.clone(), KEK, PAGE, REALM, no_history_opts()) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..400 { + w.put(format!("n{i:05}").as_bytes(), &VALUE).await.unwrap(); + } + w.commit().await.unwrap(); + } + let high_water_after = db.stats().await.unwrap().main_db_next_page_id; + assert!( + high_water_after <= high_water_before + 8, + "freed pages must be recovered from the durable free list and recycled \ + after an unclean reopen; high-water was {high_water_before} before the \ + crash and {high_water_after} after rewriting the same volume" + ); +} diff --git a/tests/fsck_deep.rs b/tests/fsck_deep.rs index ded2309..96d9f5f 100644 --- a/tests/fsck_deep.rs +++ b/tests/fsck_deep.rs @@ -102,3 +102,51 @@ async fn corrupt_page_detected() { ); } } + +#[tokio::test(flavor = "current_thread")] +async fn free_list_pages_are_accounted_not_orphans() { + // Disable history so deleted pages enter the durable free-list immediately + // (no retained snapshot pins them). + let opts = OpenOptions::default() + .with_buffer_pool_pages(64) + .with_commit_history_retain(pagedb::options::RetainPolicy::Disabled); + let db = Db::open_internal_with_options(MemVfs::new(), KEK, 4096, REALM, opts) + .await + .unwrap(); + + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..300 { + w.put(format!("k{i:05}").as_bytes(), &[7u8; 128]) + .await + .unwrap(); + } + w.commit().await.unwrap(); + } + { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..250 { + w.delete(format!("k{i:05}").as_bytes()).await.unwrap(); + } + w.commit().await.unwrap(); + } + assert!( + db.stats().await.unwrap().free_list_pending_entries > 0, + "setup should have populated the durable free-list" + ); + + let report = run_deep_walk(&db).await.unwrap(); + assert!( + report.page_issues.is_empty(), + "free-listed/chain pages must verify cleanly, got: {:?}", + report.page_issues + ); + // The free-list chain pages and the pages they track are accounted for, so + // none should be reported as orphans. + assert!( + report.orphan_page_ids.is_empty(), + "free-list pages must not be reported as orphans, got: {:?}", + report.orphan_page_ids + ); + assert!(report.is_clean(), "report should be clean"); +} diff --git a/tests/reader_stall_policy.rs b/tests/reader_stall_policy.rs index a01f0ec..8998883 100644 --- a/tests/reader_stall_policy.rs +++ b/tests/reader_stall_policy.rs @@ -124,6 +124,104 @@ async fn abort_oldest_aborts_old_reader() { drop(r2); } +/// After an unclean shutdown that leaves a *drainable* deferred-free backlog +/// behind, reopening the store must self-heal: the inherited backlog is not +/// corruption and must not brick the store. The deferred entries were tagged +/// with commit ids older than anything a freshly-opened reader pins, so the +/// first post-open commit drains them — the stall policy is evaluated against +/// the post-drain depth, so a reader opened against the reopened store is +/// neither rejected nor aborted. +/// +/// Repro of the reported failure: a daemon under write load is killed with a +/// deferred-free backlog past the stall threshold; on restart the consumer +/// holds a read txn across its schema-ensure write, and that first commit trips +/// `AbortOldest` against the backlog inherited from the dead writer — surfacing +/// as a permanent `Aborted` on every reopen. +#[tokio::test(flavor = "current_thread")] +async fn reopen_with_drainable_backlog_does_not_brick_readers() { + const THRESHOLD: u64 = 50; + let opts = || { + let mut o = OpenOptions::default() + .with_buffer_pool_pages(256) + // Disable commit history so the backlog is governed purely by the + // reader-pin floor: once the dead writer's reader is gone, every + // inherited entry is drainable. (Reclamation under a *bounded* + // history window is covered by `deferred_free_reclaim`.) + .with_commit_history_retain(pagedb::options::RetainPolicy::Disabled); + o.reader_stall_threshold_pages = THRESHOLD; + o + }; + let vfs = MemVfs::new(); + + // Phase 1: build a large *durable* deferred-free backlog. Holding a reader + // pins an old commit, forcing every subsequent commit's freed pages into + // the deferred-free queue (none are eligible to drain while that reader + // pins). Then drop everything without draining — simulating a writer killed + // mid-workload. The backlog persists on disk in the catalog. + { + let db = Db::open(vfs.clone(), KEK, 4096, REALM, opts()) + .await + .unwrap(); + { + let mut w = db.begin_write().await.unwrap(); + w.put(b"seed", b"v").await.unwrap(); + w.commit().await.unwrap(); + } + // Pin an old snapshot for the duration of the build. AbortOldest may + // flag it, but it keeps pinning until dropped, so the backlog grows. + let pin = db.begin_read().await.unwrap(); + for round in 0u32..40 { + let mut w = db.begin_write().await.unwrap(); + for i in 0u32..8 { + w.put(format!("k{round:03}_{i}").as_bytes(), &[0u8; 128]) + .await + .unwrap(); + } + w.commit().await.unwrap(); + let mut w2 = db.begin_write().await.unwrap(); + for i in 0u32..8 { + w2.put(format!("k{round:03}_{i}").as_bytes(), b"x") + .await + .unwrap(); + } + w2.commit().await.unwrap(); + } + let pending = db.stats().await.unwrap().free_list_pending_entries; + assert!( + pending > THRESHOLD, + "test setup should have built a backlog past the threshold; got {pending}" + ); + drop(pin); + // Handle drops here — backlog is durably on disk, undrained. + } + + // Phase 2: reopen (process restart) and run the consumer open pattern: a + // read txn held across a single write commit (schema ensure). The inherited + // backlog's entries are all older than the reopened reader's pin, so the + // commit drains them and the stall policy sees a depth below the threshold. + let db = Db::open(vfs.clone(), KEK, 4096, REALM, opts()) + .await + .unwrap(); + let reader = db.begin_read().await.unwrap(); + let mut w = db.begin_write().await.unwrap(); + w.put(b"schema", b"v1").await.unwrap(); + let res = w.commit().await; + assert!( + res.is_ok(), + "post-reopen commit must not be rejected by the stall policy on an \ + inherited drainable backlog: {res:?}" + ); + + let got = reader.get(b"seed").await; + assert!( + !matches!(got, Err(PagedbError::Aborted)), + "reopening a store with a drainable backlog bricked a live reader with \ + Aborted — a non-corrupt, recoverable backlog must self-heal" + ); + assert!(got.is_ok(), "reader read failed after reopen: {got:?}"); + drop(reader); +} + #[tokio::test(flavor = "current_thread")] async fn non_abortable_reader_survives_abort_oldest() { let vfs = MemVfs::new();